nydus_api/
config.rs

1// Copyright 2022 Alibaba Cloud. All rights reserved.
2// Copyright 2020 Ant Group. All rights reserved.
3//
4// SPDX-License-Identifier: Apache-2.0
5
6use std::collections::HashMap;
7use std::convert::{TryFrom, TryInto};
8use std::fs;
9use std::io::{Error, ErrorKind, Result};
10use std::path::Path;
11use std::str::FromStr;
12use std::sync::atomic::{AtomicBool, Ordering};
13use std::sync::Arc;
14
15use serde::Deserialize;
16use serde_json::Value;
17
18/// Configuration file format version 2, based on Toml.
19#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
20pub struct ConfigV2 {
21    /// Configuration file format version number, must be 2.
22    pub version: u32,
23    /// Identifier for the instance.
24    #[serde(default)]
25    pub id: String,
26    /// Configuration information for storage backend.
27    pub backend: Option<BackendConfigV2>,
28    /// Configuration for external storage backends, order insensitivity.
29    #[serde(default)]
30    pub external_backends: Vec<ExternalBackendConfig>,
31    /// Configuration information for local cache system.
32    pub cache: Option<CacheConfigV2>,
33    /// Configuration information for RAFS filesystem.
34    pub rafs: Option<RafsConfigV2>,
35    /// Overlay configuration information for the instance.
36    pub overlay: Option<OverlayConfig>,
37    /// Internal runtime configuration.
38    #[serde(skip)]
39    pub internal: ConfigV2Internal,
40}
41
42impl Default for ConfigV2 {
43    fn default() -> Self {
44        ConfigV2 {
45            version: 2,
46            id: String::new(),
47            backend: None,
48            external_backends: Vec::new(),
49            cache: None,
50            rafs: None,
51            overlay: None,
52            internal: ConfigV2Internal::default(),
53        }
54    }
55}
56
57impl ConfigV2 {
58    /// Create a new instance of `ConfigV2` object.
59    pub fn new(id: &str) -> Self {
60        ConfigV2 {
61            version: 2,
62            id: id.to_string(),
63            backend: None,
64            external_backends: Vec::new(),
65            cache: None,
66            rafs: None,
67            overlay: None,
68            internal: ConfigV2Internal::default(),
69        }
70    }
71
72    /// Create a new configuration object for `backend-localfs` and `filecache`.
73    pub fn new_localfs(id: &str, dir: &str) -> Result<Self> {
74        let content = format!(
75            r#"
76        version = 2
77        id = "{}"
78        backend.type = "localfs"
79        backend.localfs.dir = "{}"
80        cache.type = "filecache"
81        cache.compressed = false
82        cache.validate = false
83        cache.filecache.work_dir = "{}"
84        "#,
85            id, dir, dir
86        );
87
88        Self::from_str(&content)
89    }
90
91    /// Read configuration information from a file.
92    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
93        let md = fs::metadata(path.as_ref())?;
94        if md.len() > 0x100000 {
95            return Err(Error::new(
96                ErrorKind::Other,
97                "configuration file size is too big",
98            ));
99        }
100        let content = fs::read_to_string(path)?;
101        Self::from_str(&content)
102    }
103
104    /// Validate the configuration object.
105    pub fn validate(&self) -> bool {
106        if self.version != 2 {
107            return false;
108        }
109        if let Some(backend_cfg) = self.backend.as_ref() {
110            if !backend_cfg.validate() {
111                return false;
112            }
113        }
114        if let Some(cache_cfg) = self.cache.as_ref() {
115            if !cache_cfg.validate() {
116                return false;
117            }
118        }
119        if let Some(rafs_cfg) = self.rafs.as_ref() {
120            if !rafs_cfg.validate() {
121                return false;
122            }
123        }
124
125        true
126    }
127
128    /// Get configuration information for storage backend.
129    pub fn get_backend_config(&self) -> Result<&BackendConfigV2> {
130        self.backend.as_ref().ok_or_else(|| {
131            Error::new(
132                ErrorKind::InvalidInput,
133                "no configuration information for backend",
134            )
135        })
136    }
137
138    /// Get configuration information for cache subsystem.
139    pub fn get_cache_config(&self) -> Result<&CacheConfigV2> {
140        self.cache.as_ref().ok_or_else(|| {
141            Error::new(
142                ErrorKind::InvalidData,
143                "no configuration information for cache",
144            )
145        })
146    }
147
148    /// Get cache working directory.
149    pub fn get_cache_working_directory(&self) -> Result<String> {
150        let cache = self.get_cache_config()?;
151        if cache.is_filecache() {
152            if let Some(c) = cache.file_cache.as_ref() {
153                return Ok(c.work_dir.clone());
154            }
155        } else if cache.is_fscache() {
156            if let Some(c) = cache.fs_cache.as_ref() {
157                return Ok(c.work_dir.clone());
158            }
159        }
160
161        Err(Error::new(
162            ErrorKind::NotFound,
163            "no working directory configured",
164        ))
165    }
166
167    /// Get configuration information for RAFS filesystem.
168    pub fn get_rafs_config(&self) -> Result<&RafsConfigV2> {
169        self.rafs.as_ref().ok_or_else(|| {
170            Error::new(
171                ErrorKind::InvalidInput,
172                "no configuration information for rafs",
173            )
174        })
175    }
176
177    /// Clone the object with all secrets removed.
178    pub fn clone_without_secrets(&self) -> Self {
179        let mut cfg = self.clone();
180
181        if let Some(backend_cfg) = cfg.backend.as_mut() {
182            if let Some(oss_cfg) = backend_cfg.oss.as_mut() {
183                oss_cfg.access_key_id = String::new();
184                oss_cfg.access_key_secret = String::new();
185            }
186            if let Some(registry_cfg) = backend_cfg.registry.as_mut() {
187                registry_cfg.auth = None;
188                registry_cfg.registry_token = None;
189            }
190        }
191
192        cfg
193    }
194
195    /// Check whether chunk digest validation is enabled or not.
196    pub fn is_chunk_validation_enabled(&self) -> bool {
197        let mut validation = if let Some(cache) = &self.cache {
198            cache.cache_validate
199        } else {
200            false
201        };
202        if let Some(rafs) = &self.rafs {
203            if rafs.validate {
204                validation = true;
205            }
206        }
207
208        validation
209    }
210
211    /// Check whether fscache is enabled or not.
212    pub fn is_fs_cache(&self) -> bool {
213        if let Some(cache) = self.cache.as_ref() {
214            cache.fs_cache.is_some()
215        } else {
216            false
217        }
218    }
219
220    /// Fill authorization for registry backend.
221    pub fn update_registry_auth_info(&mut self, auth: &Option<String>) {
222        if let Some(auth) = auth {
223            if let Some(backend) = self.backend.as_mut() {
224                if let Some(registry) = backend.registry.as_mut() {
225                    registry.auth = Some(auth.to_string());
226                }
227            }
228        }
229    }
230}
231
232impl FromStr for ConfigV2 {
233    type Err = std::io::Error;
234
235    fn from_str(s: &str) -> Result<ConfigV2> {
236        if let Ok(v) = serde_json::from_str::<ConfigV2>(s) {
237            return if v.validate() {
238                Ok(v)
239            } else {
240                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
241            };
242        }
243        if let Ok(v) = toml::from_str::<ConfigV2>(s) {
244            return if v.validate() {
245                Ok(v)
246            } else {
247                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
248            };
249        }
250        if let Ok(v) = serde_json::from_str::<RafsConfig>(s) {
251            if let Ok(v) = ConfigV2::try_from(v) {
252                if v.validate() {
253                    return Ok(v);
254                }
255            }
256        }
257        Err(Error::new(
258            ErrorKind::InvalidInput,
259            "failed to parse configuration information",
260        ))
261    }
262}
263
264/// Configuration information for storage backend.
265#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
266pub struct BackendConfigV2 {
267    /// Type of storage backend.
268    #[serde(rename = "type")]
269    pub backend_type: String,
270    /// Configuration for local disk backend.
271    pub localdisk: Option<LocalDiskConfig>,
272    /// Configuration for local filesystem backend.
273    pub localfs: Option<LocalFsConfig>,
274    /// Configuration for OSS backend.
275    pub oss: Option<OssConfig>,
276    /// Configuration for S3 backend.
277    pub s3: Option<S3Config>,
278    /// Configuration for container registry backend.
279    pub registry: Option<RegistryConfig>,
280    /// Configuration for local http proxy.
281    #[serde(rename = "http-proxy")]
282    pub http_proxy: Option<HttpProxyConfig>,
283}
284
285impl BackendConfigV2 {
286    /// Validate storage backend configuration.
287    pub fn validate(&self) -> bool {
288        match self.backend_type.as_str() {
289            "localdisk" => match self.localdisk.as_ref() {
290                Some(v) => {
291                    if v.device_path.is_empty() {
292                        return false;
293                    }
294                }
295                None => return false,
296            },
297            "localfs" => match self.localfs.as_ref() {
298                Some(v) => {
299                    if v.blob_file.is_empty() && v.dir.is_empty() {
300                        return false;
301                    }
302                }
303                None => return false,
304            },
305            "oss" => match self.oss.as_ref() {
306                Some(v) => {
307                    if v.endpoint.is_empty() || v.bucket_name.is_empty() {
308                        return false;
309                    }
310                }
311                None => return false,
312            },
313            "s3" => match self.s3.as_ref() {
314                Some(v) => {
315                    if v.region.is_empty() || v.bucket_name.is_empty() {
316                        return false;
317                    }
318                }
319                None => return false,
320            },
321            "registry" => match self.registry.as_ref() {
322                Some(v) => {
323                    if v.host.is_empty() || v.repo.is_empty() {
324                        return false;
325                    }
326                }
327                None => return false,
328            },
329
330            "http-proxy" => match self.http_proxy.as_ref() {
331                Some(v) => {
332                    let is_valid_unix_socket_path = |path: &str| {
333                        let path = Path::new(path);
334                        path.is_absolute() && path.exists()
335                    };
336                    if v.addr.is_empty()
337                        || !(v.addr.starts_with("http://")
338                            || v.addr.starts_with("https://")
339                            || is_valid_unix_socket_path(&v.addr))
340                    {
341                        return false;
342                    }
343
344                    // check if v.path is valid url path format
345                    if Path::new(&v.path).join("any_blob_id").to_str().is_none() {
346                        return false;
347                    }
348                }
349                None => return false,
350            },
351            _ => return false,
352        }
353
354        true
355    }
356
357    /// Get configuration information for localdisk
358    pub fn get_localdisk_config(&self) -> Result<&LocalDiskConfig> {
359        if &self.backend_type != "localdisk" {
360            Err(Error::new(
361                ErrorKind::InvalidInput,
362                "backend type is not 'localdisk'",
363            ))
364        } else {
365            self.localdisk.as_ref().ok_or_else(|| {
366                Error::new(
367                    ErrorKind::InvalidData,
368                    "no configuration information for localdisk",
369                )
370            })
371        }
372    }
373
374    /// Get configuration information for localfs
375    pub fn get_localfs_config(&self) -> Result<&LocalFsConfig> {
376        if &self.backend_type != "localfs" {
377            Err(Error::new(
378                ErrorKind::InvalidInput,
379                "backend type is not 'localfs'",
380            ))
381        } else {
382            self.localfs.as_ref().ok_or_else(|| {
383                Error::new(
384                    ErrorKind::InvalidData,
385                    "no configuration information for localfs",
386                )
387            })
388        }
389    }
390
391    /// Get configuration information for OSS
392    pub fn get_oss_config(&self) -> Result<&OssConfig> {
393        if &self.backend_type != "oss" {
394            Err(Error::new(
395                ErrorKind::InvalidInput,
396                "backend type is not 'oss'",
397            ))
398        } else {
399            self.oss.as_ref().ok_or_else(|| {
400                Error::new(
401                    ErrorKind::InvalidData,
402                    "no configuration information for OSS",
403                )
404            })
405        }
406    }
407
408    /// Get configuration information for S3
409    pub fn get_s3_config(&self) -> Result<&S3Config> {
410        if &self.backend_type != "s3" {
411            Err(Error::new(
412                ErrorKind::InvalidInput,
413                "backend type is not 's3'",
414            ))
415        } else {
416            self.s3.as_ref().ok_or_else(|| {
417                Error::new(
418                    ErrorKind::InvalidData,
419                    "no configuration information for s3",
420                )
421            })
422        }
423    }
424
425    /// Get configuration information for Registry
426    pub fn get_registry_config(&self) -> Result<&RegistryConfig> {
427        if &self.backend_type != "registry" {
428            Err(Error::new(
429                ErrorKind::InvalidInput,
430                "backend type is not 'registry'",
431            ))
432        } else {
433            self.registry.as_ref().ok_or_else(|| {
434                Error::new(
435                    ErrorKind::InvalidData,
436                    "no configuration information for registry",
437                )
438            })
439        }
440    }
441
442    /// Get configuration information for http proxy
443    pub fn get_http_proxy_config(&self) -> Result<&HttpProxyConfig> {
444        if &self.backend_type != "http-proxy" {
445            Err(Error::new(
446                ErrorKind::InvalidInput,
447                "backend type is not 'http-proxy'",
448            ))
449        } else {
450            self.http_proxy.as_ref().ok_or_else(|| {
451                Error::new(
452                    ErrorKind::InvalidData,
453                    "no configuration information for http-proxy",
454                )
455            })
456        }
457    }
458}
459
460/// Configuration information for localdisk storage backend.
461#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
462pub struct LocalDiskConfig {
463    /// Mounted block device path or original localdisk image file path.
464    #[serde(default)]
465    pub device_path: String,
466    /// Disable discover blob objects by scanning GPT table.
467    #[serde(default)]
468    pub disable_gpt: bool,
469}
470
471/// Configuration information for localfs storage backend.
472#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
473pub struct LocalFsConfig {
474    /// Blob file to access.
475    #[serde(default)]
476    pub blob_file: String,
477    /// Dir to hold blob files. Used when 'blob_file' is not specified.
478    #[serde(default)]
479    pub dir: String,
480    /// Alternative dirs to search for blobs.
481    #[serde(default)]
482    pub alt_dirs: Vec<String>,
483}
484
485/// OSS configuration information to access blobs.
486#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
487pub struct OssConfig {
488    /// Oss http scheme, either 'http' or 'https'
489    #[serde(default = "default_http_scheme")]
490    pub scheme: String,
491    /// Oss endpoint
492    pub endpoint: String,
493    /// Oss bucket name
494    pub bucket_name: String,
495    /// Prefix object_prefix to OSS object key, for example the simulation of subdirectory:
496    /// - object_key: sha256:xxx
497    /// - object_prefix: nydus/
498    /// - object_key with object_prefix: nydus/sha256:xxx
499    #[serde(default)]
500    pub object_prefix: String,
501    /// Oss access key
502    #[serde(default)]
503    pub access_key_id: String,
504    /// Oss secret
505    #[serde(default)]
506    pub access_key_secret: String,
507    /// Skip SSL certificate validation for HTTPS scheme.
508    #[serde(default)]
509    pub skip_verify: bool,
510    /// Drop the read request once http request timeout, in seconds.
511    #[serde(default = "default_http_timeout")]
512    pub timeout: u32,
513    /// Drop the read request once http connection timeout, in seconds.
514    #[serde(default = "default_http_timeout")]
515    pub connect_timeout: u32,
516    /// Retry count when read request failed.
517    #[serde(default)]
518    pub retry_limit: u8,
519    /// Enable HTTP proxy for the read request.
520    #[serde(default)]
521    pub proxy: ProxyConfig,
522    /// Enable mirrors for the read request.
523    #[serde(default)]
524    pub mirrors: Vec<MirrorConfig>,
525}
526
527/// S3 configuration information to access blobs.
528#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
529pub struct S3Config {
530    /// S3 http scheme, either 'http' or 'https'
531    #[serde(default = "default_http_scheme")]
532    pub scheme: String,
533    /// S3 endpoint
534    pub endpoint: String,
535    /// S3 region
536    pub region: String,
537    /// S3 bucket name
538    pub bucket_name: String,
539    /// Prefix object_prefix to S3 object key, for example the simulation of subdirectory:
540    /// - object_key: sha256:xxx
541    /// - object_prefix: nydus/
542    /// - object_key with object_prefix: nydus/sha256:xxx
543    #[serde(default)]
544    pub object_prefix: String,
545    /// S3 access key
546    #[serde(default)]
547    pub access_key_id: String,
548    /// S3 secret
549    #[serde(default)]
550    pub access_key_secret: String,
551    /// Skip SSL certificate validation for HTTPS scheme.
552    #[serde(default)]
553    pub skip_verify: bool,
554    /// Drop the read request once http request timeout, in seconds.
555    #[serde(default = "default_http_timeout")]
556    pub timeout: u32,
557    /// Drop the read request once http connection timeout, in seconds.
558    #[serde(default = "default_http_timeout")]
559    pub connect_timeout: u32,
560    /// Retry count when read request failed.
561    #[serde(default)]
562    pub retry_limit: u8,
563    /// Enable HTTP proxy for the read request.
564    #[serde(default)]
565    pub proxy: ProxyConfig,
566    /// Enable mirrors for the read request.
567    #[serde(default)]
568    pub mirrors: Vec<MirrorConfig>,
569}
570
571/// Http proxy configuration information to access blobs.
572#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
573pub struct HttpProxyConfig {
574    /// Address of http proxy server, like `http://xxx.xxx` or `https://xxx.xxx` or `/path/to/unix.sock`.
575    pub addr: String,
576    /// Path to access the blobs, like `/<_namespace>/<_repo>/blobs`.
577    /// If the http proxy server is over unix socket, this field will be ignored.
578    #[serde(default)]
579    pub path: String,
580    /// Skip SSL certificate validation for HTTPS scheme.
581    #[serde(default)]
582    pub skip_verify: bool,
583    /// Drop the read request once http request timeout, in seconds.
584    #[serde(default = "default_http_timeout")]
585    pub timeout: u32,
586    /// Drop the read request once http connection timeout, in seconds.
587    #[serde(default = "default_http_timeout")]
588    pub connect_timeout: u32,
589    /// Retry count when read request failed.
590    #[serde(default)]
591    pub retry_limit: u8,
592    /// Enable HTTP proxy for the read request.
593    #[serde(default)]
594    pub proxy: ProxyConfig,
595    /// Enable mirrors for the read request.
596    #[serde(default)]
597    pub mirrors: Vec<MirrorConfig>,
598}
599
600/// Container registry configuration information to access blobs.
601#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
602pub struct RegistryConfig {
603    /// Registry http scheme, either 'http' or 'https'
604    #[serde(default = "default_http_scheme")]
605    pub scheme: String,
606    /// Registry url host
607    pub host: String,
608    /// Registry image name, like 'library/ubuntu'
609    pub repo: String,
610    /// Base64_encoded(username:password), the field should be sent to registry auth server to get a bearer token.
611    #[serde(default)]
612    pub auth: Option<String>,
613    /// Skip SSL certificate validation for HTTPS scheme.
614    #[serde(default)]
615    pub skip_verify: bool,
616    /// Drop the read request once http request timeout, in seconds.
617    #[serde(default = "default_http_timeout")]
618    pub timeout: u32,
619    /// Drop the read request once http connection timeout, in seconds.
620    #[serde(default = "default_http_timeout")]
621    pub connect_timeout: u32,
622    /// Retry count when read request failed.
623    #[serde(default)]
624    pub retry_limit: u8,
625    /// The field is a bearer token to be sent to registry to authorize registry requests.
626    #[serde(default)]
627    pub registry_token: Option<String>,
628    /// The http scheme to access blobs. It is used to workaround some P2P subsystem
629    /// that requires a different scheme than the registry.
630    #[serde(default)]
631    pub blob_url_scheme: String,
632    /// Redirect blob access to a different host regardless of the one specified in 'host'.
633    #[serde(default)]
634    pub blob_redirected_host: String,
635    /// Enable HTTP proxy for the read request.
636    #[serde(default)]
637    pub proxy: ProxyConfig,
638    /// Enable mirrors for the read request.
639    #[serde(default)]
640    pub mirrors: Vec<MirrorConfig>,
641}
642
643/// Configuration information for blob cache manager.
644#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
645pub struct CacheConfigV2 {
646    /// Type of blob cache: "blobcache", "fscache" or "dummy"
647    #[serde(default, rename = "type")]
648    pub cache_type: String,
649    /// Whether the data from the cache is compressed, not used anymore.
650    #[serde(default, rename = "compressed")]
651    pub cache_compressed: bool,
652    /// Whether to validate data read from the cache.
653    #[serde(default, rename = "validate")]
654    pub cache_validate: bool,
655    /// Configuration for blob level prefetch.
656    #[serde(default)]
657    pub prefetch: PrefetchConfigV2,
658    /// Configuration information for file cache
659    #[serde(rename = "filecache")]
660    pub file_cache: Option<FileCacheConfig>,
661    #[serde(rename = "fscache")]
662    /// Configuration information for fscache
663    pub fs_cache: Option<FsCacheConfig>,
664}
665
666impl CacheConfigV2 {
667    /// Validate cache configuration information.
668    pub fn validate(&self) -> bool {
669        match self.cache_type.as_str() {
670            "blobcache" | "filecache" => {
671                if let Some(c) = self.file_cache.as_ref() {
672                    if c.work_dir.is_empty() {
673                        return false;
674                    }
675                } else {
676                    return false;
677                }
678            }
679            "fscache" => {
680                if let Some(c) = self.fs_cache.as_ref() {
681                    if c.work_dir.is_empty() {
682                        return false;
683                    }
684                } else {
685                    return false;
686                }
687            }
688            "" | "dummycache" => {}
689            _ => return false,
690        }
691
692        if self.prefetch.enable {
693            if self.prefetch.batch_size > 0x10000000 {
694                return false;
695            }
696            if self.prefetch.threads_count == 0 || self.prefetch.threads_count > 1024 {
697                return false;
698            }
699        }
700
701        true
702    }
703
704    /// Check whether the cache type is `filecache`
705    pub fn is_filecache(&self) -> bool {
706        self.cache_type == "blobcache" || self.cache_type == "filecache"
707    }
708
709    /// Check whether the cache type is `fscache`
710    pub fn is_fscache(&self) -> bool {
711        self.cache_type == "fscache"
712    }
713
714    /// Get configuration information for file cache.
715    pub fn get_filecache_config(&self) -> Result<&FileCacheConfig> {
716        if self.is_filecache() {
717            self.file_cache.as_ref().ok_or_else(|| {
718                Error::new(
719                    ErrorKind::InvalidInput,
720                    "no configuration information for filecache",
721                )
722            })
723        } else {
724            Err(Error::new(
725                ErrorKind::InvalidData,
726                "cache type is not 'filecache'",
727            ))
728        }
729    }
730
731    /// Get configuration information for fscache.
732    pub fn get_fscache_config(&self) -> Result<&FsCacheConfig> {
733        if self.is_fscache() {
734            self.fs_cache.as_ref().ok_or_else(|| {
735                Error::new(
736                    ErrorKind::InvalidData,
737                    "no configuration information for fscache",
738                )
739            })
740        } else {
741            Err(Error::new(
742                ErrorKind::InvalidInput,
743                "cache type is not 'fscache'",
744            ))
745        }
746    }
747}
748
749/// Configuration information for file cache.
750#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
751pub struct FileCacheConfig {
752    /// Working directory to store state and cached files.
753    #[serde(default = "default_work_dir")]
754    pub work_dir: String,
755    /// Deprecated: disable index mapping, keep it as false when possible.
756    #[serde(default)]
757    pub disable_indexed_map: bool,
758    /// Enable encryption data written to the cache file.
759    #[serde(default)]
760    pub enable_encryption: bool,
761    /// Enable convergent encryption for chunk deduplication.
762    #[serde(default)]
763    pub enable_convergent_encryption: bool,
764    /// Key for data encryption, a heximal representation of [u8; 32].
765    #[serde(default)]
766    pub encryption_key: String,
767}
768
769impl FileCacheConfig {
770    /// Get the working directory.
771    pub fn get_work_dir(&self) -> Result<&str> {
772        let path = fs::metadata(&self.work_dir)
773            .or_else(|_| {
774                fs::create_dir_all(&self.work_dir)?;
775                fs::metadata(&self.work_dir)
776            })
777            .map_err(|e| {
778                log::error!("fail to stat filecache work_dir {}: {}", self.work_dir, e);
779                e
780            })?;
781
782        if path.is_dir() {
783            Ok(&self.work_dir)
784        } else {
785            Err(Error::new(
786                ErrorKind::NotFound,
787                format!("filecache work_dir {} is not a directory", self.work_dir),
788            ))
789        }
790    }
791}
792
793/// Configuration information for fscache.
794#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
795pub struct FsCacheConfig {
796    /// Working directory to store state and cached files.
797    #[serde(default = "default_work_dir")]
798    pub work_dir: String,
799}
800
801impl FsCacheConfig {
802    /// Get the working directory.
803    pub fn get_work_dir(&self) -> Result<&str> {
804        let path = fs::metadata(&self.work_dir)
805            .or_else(|_| {
806                fs::create_dir_all(&self.work_dir)?;
807                fs::metadata(&self.work_dir)
808            })
809            .map_err(|e| {
810                log::error!("fail to stat fscache work_dir {}: {}", self.work_dir, e);
811                e
812            })?;
813
814        if path.is_dir() {
815            Ok(&self.work_dir)
816        } else {
817            Err(Error::new(
818                ErrorKind::NotFound,
819                format!("fscache work_dir {} is not a directory", self.work_dir),
820            ))
821        }
822    }
823}
824
825/// Configuration information for RAFS filesystem.
826#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
827pub struct RafsConfigV2 {
828    /// Filesystem metadata cache mode.
829    #[serde(default = "default_rafs_mode")]
830    pub mode: String,
831    /// Amplified user IO request batch size to read data from remote storage backend / local cache.
832    #[serde(rename = "batch_size", default = "default_user_io_batch_size")]
833    pub user_io_batch_size: usize,
834    /// Whether to validate data digest.
835    #[serde(default)]
836    pub validate: bool,
837    /// Enable support of extended attributes.
838    #[serde(default)]
839    pub enable_xattr: bool,
840    /// Record file operation metrics for each file.
841    ///
842    /// Better to keep it off in production environment due to possible resource consumption.
843    #[serde(default)]
844    pub iostats_files: bool,
845    /// Record filesystem access pattern.
846    #[serde(default)]
847    pub access_pattern: bool,
848    /// Record file name if file access trace log.
849    #[serde(default)]
850    pub latest_read_files: bool,
851    /// Filesystem prefetching configuration.
852    #[serde(default)]
853    pub prefetch: PrefetchConfigV2,
854}
855
856impl RafsConfigV2 {
857    /// Validate RAFS filesystem configuration information.
858    pub fn validate(&self) -> bool {
859        if self.mode != "direct" && self.mode != "cached" {
860            return false;
861        }
862        if self.user_io_batch_size > 0x10000000 {
863            return false;
864        }
865        if self.prefetch.enable {
866            if self.prefetch.batch_size > 0x10000000 {
867                return false;
868            }
869            if self.prefetch.threads_count == 0 || self.prefetch.threads_count > 1024 {
870                return false;
871            }
872        }
873
874        true
875    }
876}
877
878/// Configuration information for blob data prefetching.
879#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
880pub struct PrefetchConfigV2 {
881    /// Whether to enable blob data prefetching.
882    pub enable: bool,
883    /// Number of data prefetching working threads.
884    #[serde(rename = "threads", default = "default_prefetch_threads_count")]
885    pub threads_count: usize,
886    /// The amplify batch size to prefetch data from backend.
887    #[serde(default = "default_prefetch_batch_size")]
888    pub batch_size: usize,
889    /// Network bandwidth rate limit in unit of Bytes and Zero means no limit.
890    #[serde(default)]
891    pub bandwidth_limit: u32,
892    /// Prefetch all data from backend.
893    #[serde(default)]
894    pub prefetch_all: bool,
895}
896
897/// Configuration information for network proxy.
898#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
899pub struct ProxyConfig {
900    /// Access remote storage backend via proxy, e.g. Dragonfly dfdaemon server URL.
901    #[serde(default)]
902    pub url: String,
903    /// Proxy health checking endpoint.
904    #[serde(default)]
905    pub ping_url: String,
906    /// Fallback to remote storage backend if proxy ping failed.
907    #[serde(default = "default_true")]
908    pub fallback: bool,
909    /// Interval for proxy health checking, in seconds.
910    #[serde(default = "default_check_interval")]
911    pub check_interval: u64,
912    /// Replace URL to http to request source registry with proxy, and allow fallback to https if the proxy is unhealthy.
913    #[serde(default)]
914    pub use_http: bool,
915    /// Elapsed time to pause proxy health check when the request is inactive, in seconds.
916    #[serde(default = "default_check_pause_elapsed")]
917    pub check_pause_elapsed: u64,
918}
919
920impl Default for ProxyConfig {
921    fn default() -> Self {
922        Self {
923            url: String::new(),
924            ping_url: String::new(),
925            fallback: true,
926            check_interval: 5,
927            use_http: false,
928            check_pause_elapsed: 300,
929        }
930    }
931}
932
933/// Configuration for registry mirror.
934#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
935pub struct MirrorConfig {
936    /// Mirror server URL, for example http://127.0.0.1:65001.
937    pub host: String,
938    /// Ping URL to check mirror server health.
939    #[serde(default)]
940    pub ping_url: String,
941    /// HTTP request headers to be passed to mirror server.
942    #[serde(default)]
943    pub headers: HashMap<String, String>,
944    /// Interval for mirror health checking, in seconds.
945    #[serde(default = "default_check_interval")]
946    pub health_check_interval: u64,
947    /// Maximum number of failures before marking a mirror as unusable.
948    #[serde(default = "default_failure_limit")]
949    pub failure_limit: u8,
950    /// Elapsed time to pause mirror health check when the request is inactive, in seconds.
951    #[serde(default = "default_check_pause_elapsed")]
952    pub health_check_pause_elapsed: u64,
953}
954
955impl Default for MirrorConfig {
956    fn default() -> Self {
957        Self {
958            host: String::new(),
959            headers: HashMap::new(),
960            health_check_interval: 5,
961            failure_limit: 5,
962            ping_url: String::new(),
963            health_check_pause_elapsed: 300,
964        }
965    }
966}
967
968/// Configuration information for a cached blob`.
969#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
970pub struct BlobCacheEntryConfigV2 {
971    /// Configuration file format version number, must be 2.
972    pub version: u32,
973    /// Identifier for the instance.
974    #[serde(default)]
975    pub id: String,
976    /// Configuration information for storage backend.
977    #[serde(default)]
978    pub backend: BackendConfigV2,
979    /// Configuration for external storage backends, order insensitivity.
980    #[serde(default)]
981    pub external_backends: Vec<ExternalBackendConfig>,
982    /// Configuration information for local cache system.
983    #[serde(default)]
984    pub cache: CacheConfigV2,
985    /// Optional file path for metadata blob.
986    #[serde(default)]
987    pub metadata_path: Option<String>,
988}
989
990impl BlobCacheEntryConfigV2 {
991    /// Read configuration information from a file.
992    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
993        let md = fs::metadata(path.as_ref())?;
994        if md.len() > 0x100000 {
995            return Err(Error::new(
996                ErrorKind::InvalidInput,
997                "configuration file size is too big",
998            ));
999        }
1000        let content = fs::read_to_string(path)?;
1001        Self::from_str(&content)
1002    }
1003
1004    /// Validate the configuration object.
1005    pub fn validate(&self) -> bool {
1006        if self.version != 2 {
1007            return false;
1008        }
1009        let config: ConfigV2 = self.into();
1010        config.validate()
1011    }
1012}
1013
1014impl FromStr for BlobCacheEntryConfigV2 {
1015    type Err = Error;
1016
1017    fn from_str(s: &str) -> Result<BlobCacheEntryConfigV2> {
1018        if let Ok(v) = serde_json::from_str::<BlobCacheEntryConfigV2>(s) {
1019            return if v.validate() {
1020                Ok(v)
1021            } else {
1022                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
1023            };
1024        }
1025        if let Ok(v) = toml::from_str::<BlobCacheEntryConfigV2>(s) {
1026            return if v.validate() {
1027                Ok(v)
1028            } else {
1029                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
1030            };
1031        }
1032        Err(Error::new(
1033            ErrorKind::InvalidInput,
1034            "failed to parse configuration information",
1035        ))
1036    }
1037}
1038
1039impl From<&BlobCacheEntryConfigV2> for ConfigV2 {
1040    fn from(c: &BlobCacheEntryConfigV2) -> Self {
1041        ConfigV2 {
1042            version: c.version,
1043            id: c.id.clone(),
1044            backend: Some(c.backend.clone()),
1045            external_backends: c.external_backends.clone(),
1046            cache: Some(c.cache.clone()),
1047            rafs: None,
1048            overlay: None,
1049            internal: ConfigV2Internal::default(),
1050        }
1051    }
1052}
1053
1054/// Internal runtime configuration.
1055#[derive(Clone, Debug)]
1056pub struct ConfigV2Internal {
1057    /// It's possible to access the raw or more blob objects.
1058    pub blob_accessible: Arc<AtomicBool>,
1059}
1060
1061impl Default for ConfigV2Internal {
1062    fn default() -> Self {
1063        ConfigV2Internal {
1064            blob_accessible: Arc::new(AtomicBool::new(false)),
1065        }
1066    }
1067}
1068
1069impl PartialEq for ConfigV2Internal {
1070    fn eq(&self, other: &Self) -> bool {
1071        self.blob_accessible() == other.blob_accessible()
1072    }
1073}
1074
1075impl Eq for ConfigV2Internal {}
1076
1077impl ConfigV2Internal {
1078    /// Get the auto-probe flag.
1079    pub fn blob_accessible(&self) -> bool {
1080        self.blob_accessible.load(Ordering::Relaxed)
1081    }
1082
1083    /// Set the auto-probe flag.
1084    pub fn set_blob_accessible(&self, accessible: bool) {
1085        self.blob_accessible.store(accessible, Ordering::Relaxed);
1086    }
1087}
1088
1089/// Blob cache object type for nydus/rafs bootstrap blob.
1090pub const BLOB_CACHE_TYPE_META_BLOB: &str = "bootstrap";
1091/// Blob cache object type for nydus/rafs data blob.
1092pub const BLOB_CACHE_TYPE_DATA_BLOB: &str = "datablob";
1093
1094/// Configuration information for a cached blob.
1095#[derive(Debug, Deserialize, Serialize, Clone)]
1096pub struct BlobCacheEntry {
1097    /// Type of blob object, bootstrap or data blob.
1098    #[serde(rename = "type")]
1099    pub blob_type: String,
1100    /// Blob id.
1101    #[serde(rename = "id")]
1102    pub blob_id: String,
1103    /// Configuration information to generate blob cache object.
1104    #[serde(default, rename = "config")]
1105    pub(crate) blob_config_legacy: Option<BlobCacheEntryConfig>,
1106    /// Configuration information to generate blob cache object.
1107    #[serde(default, rename = "config_v2")]
1108    pub blob_config: Option<BlobCacheEntryConfigV2>,
1109    /// Domain id for the blob, which is used to group cached blobs into management domains.
1110    #[serde(default)]
1111    pub domain_id: String,
1112}
1113
1114impl BlobCacheEntry {
1115    pub fn prepare_configuration_info(&mut self) -> bool {
1116        if self.blob_config.is_none() {
1117            if let Some(legacy) = self.blob_config_legacy.as_ref() {
1118                match legacy.try_into() {
1119                    Err(_) => return false,
1120                    Ok(v) => self.blob_config = Some(v),
1121                }
1122            }
1123        }
1124
1125        match self.blob_config.as_ref() {
1126            None => false,
1127            Some(cfg) => cfg.cache.validate() && cfg.backend.validate(),
1128        }
1129    }
1130}
1131
1132impl BlobCacheEntry {
1133    /// Read configuration information from a file.
1134    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
1135        let md = fs::metadata(path.as_ref())?;
1136        if md.len() > 0x100000 {
1137            return Err(Error::new(
1138                ErrorKind::InvalidInput,
1139                "configuration file size is too big",
1140            ));
1141        }
1142        let content = fs::read_to_string(path)?;
1143        Self::from_str(&content)
1144    }
1145
1146    /// Validate the configuration object.
1147    pub fn validate(&self) -> bool {
1148        if self.blob_type != BLOB_CACHE_TYPE_META_BLOB
1149            && self.blob_type != BLOB_CACHE_TYPE_DATA_BLOB
1150        {
1151            log::warn!("invalid blob type {} for blob cache entry", self.blob_type);
1152            return false;
1153        }
1154        if let Some(config) = self.blob_config.as_ref() {
1155            if !config.validate() {
1156                return false;
1157            }
1158        }
1159        true
1160    }
1161}
1162
1163impl FromStr for BlobCacheEntry {
1164    type Err = Error;
1165
1166    fn from_str(s: &str) -> Result<BlobCacheEntry> {
1167        if let Ok(v) = serde_json::from_str::<BlobCacheEntry>(s) {
1168            return if v.validate() {
1169                Ok(v)
1170            } else {
1171                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
1172            };
1173        }
1174        if let Ok(v) = toml::from_str::<BlobCacheEntry>(s) {
1175            return if v.validate() {
1176                Ok(v)
1177            } else {
1178                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
1179            };
1180        }
1181        Err(Error::new(
1182            ErrorKind::InvalidInput,
1183            "failed to parse configuration information",
1184        ))
1185    }
1186}
1187
1188/// Configuration information for a list of cached blob objects.
1189#[derive(Debug, Default, Deserialize, Serialize)]
1190pub struct BlobCacheList {
1191    /// List of blob configuration information.
1192    pub blobs: Vec<BlobCacheEntry>,
1193}
1194
1195fn default_true() -> bool {
1196    true
1197}
1198
1199fn default_http_scheme() -> String {
1200    "https".to_string()
1201}
1202
1203fn default_http_timeout() -> u32 {
1204    5
1205}
1206
1207fn default_check_interval() -> u64 {
1208    5
1209}
1210
1211fn default_check_pause_elapsed() -> u64 {
1212    300
1213}
1214
1215fn default_failure_limit() -> u8 {
1216    5
1217}
1218
1219fn default_work_dir() -> String {
1220    ".".to_string()
1221}
1222
1223pub fn default_user_io_batch_size() -> usize {
1224    1024 * 1024
1225}
1226
1227pub fn default_prefetch_batch_size() -> usize {
1228    1024 * 1024
1229}
1230
1231fn default_prefetch_threads_count() -> usize {
1232    8
1233}
1234
1235fn default_prefetch_all() -> bool {
1236    true
1237}
1238
1239fn default_rafs_mode() -> String {
1240    "direct".to_string()
1241}
1242
1243////////////////////////////////////////////////////////////////////////////////////////////////////
1244// For backward compatibility
1245////////////////////////////////////////////////////////////////////////////////////////////////////
1246
1247/// Configuration information for storage backend.
1248#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1249struct BackendConfig {
1250    /// Type of storage backend.
1251    #[serde(rename = "type")]
1252    pub backend_type: String,
1253    /// Configuration for storage backend.
1254    /// Possible value: `LocalFsConfig`, `RegistryConfig`, `OssConfig`, `LocalDiskConfig`.
1255    #[serde(rename = "config")]
1256    pub backend_config: Value,
1257}
1258
1259impl TryFrom<&BackendConfig> for BackendConfigV2 {
1260    type Error = std::io::Error;
1261
1262    fn try_from(value: &BackendConfig) -> std::result::Result<Self, Self::Error> {
1263        let mut config = BackendConfigV2 {
1264            backend_type: value.backend_type.clone(),
1265            localdisk: None,
1266            localfs: None,
1267            oss: None,
1268            s3: None,
1269            registry: None,
1270            http_proxy: None,
1271        };
1272
1273        match value.backend_type.as_str() {
1274            "localdisk" => {
1275                config.localdisk = Some(serde_json::from_value(value.backend_config.clone())?);
1276            }
1277            "localfs" => {
1278                config.localfs = Some(serde_json::from_value(value.backend_config.clone())?);
1279            }
1280            "oss" => {
1281                config.oss = Some(serde_json::from_value(value.backend_config.clone())?);
1282            }
1283            "s3" => {
1284                config.s3 = Some(serde_json::from_value(value.backend_config.clone())?);
1285            }
1286            "registry" => {
1287                config.registry = Some(serde_json::from_value(value.backend_config.clone())?);
1288            }
1289            v => {
1290                return Err(Error::new(
1291                    ErrorKind::InvalidInput,
1292                    format!("unsupported backend type '{}'", v),
1293                ))
1294            }
1295        }
1296
1297        Ok(config)
1298    }
1299}
1300
1301/// Configuration information for blob cache manager.
1302#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1303struct CacheConfig {
1304    /// Type of blob cache: "blobcache", "fscache" or ""
1305    #[serde(default, rename = "type")]
1306    pub cache_type: String,
1307    /// Whether the data from the cache is compressed, not used anymore.
1308    #[serde(default, rename = "compressed")]
1309    pub cache_compressed: bool,
1310    /// Blob cache manager specific configuration: FileCacheConfig, FsCacheConfig.
1311    #[serde(default, rename = "config")]
1312    pub cache_config: Value,
1313    /// Whether to validate data read from the cache.
1314    #[serde(default, rename = "validate")]
1315    pub cache_validate: bool,
1316    /// Configuration for blob data prefetching.
1317    #[serde(skip_serializing, skip_deserializing)]
1318    pub prefetch_config: BlobPrefetchConfig,
1319}
1320
1321/// Additional configuration information for external backend, its items
1322/// will be merged to the configuration from image.
1323#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1324pub struct ExternalBackendConfig {
1325    /// External backend identifier to merge.
1326    pub patch: HashMap<String, String>,
1327    /// External backend type.
1328    #[serde(rename = "type")]
1329    pub kind: String,
1330    /// External backend config items to merge.
1331    pub config: HashMap<String, String>,
1332}
1333
1334impl TryFrom<&CacheConfig> for CacheConfigV2 {
1335    type Error = std::io::Error;
1336
1337    fn try_from(v: &CacheConfig) -> std::result::Result<Self, Self::Error> {
1338        let mut config = CacheConfigV2 {
1339            cache_type: v.cache_type.clone(),
1340            cache_compressed: v.cache_compressed,
1341            cache_validate: v.cache_validate,
1342            prefetch: (&v.prefetch_config).into(),
1343            file_cache: None,
1344            fs_cache: None,
1345        };
1346
1347        match v.cache_type.as_str() {
1348            "blobcache" | "filecache" => {
1349                config.file_cache = Some(serde_json::from_value(v.cache_config.clone())?);
1350            }
1351            "fscache" => {
1352                config.fs_cache = Some(serde_json::from_value(v.cache_config.clone())?);
1353            }
1354            "" | "dummycache" => {}
1355            t => {
1356                return Err(Error::new(
1357                    ErrorKind::InvalidInput,
1358                    format!("unsupported cache type '{}'", t),
1359                ))
1360            }
1361        }
1362
1363        Ok(config)
1364    }
1365}
1366
1367/// Configuration information to create blob cache manager.
1368#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1369struct FactoryConfig {
1370    /// Id of the factory.
1371    #[serde(default)]
1372    pub id: String,
1373    /// Configuration for storage backend.
1374    pub backend: BackendConfig,
1375    /// Configuration for external storage backends, order insensitivity.
1376    #[serde(default)]
1377    pub external_backends: Vec<ExternalBackendConfig>,
1378    /// Configuration for blob cache manager.
1379    #[serde(default)]
1380    pub cache: CacheConfig,
1381}
1382
1383/// Rafs storage backend configuration information.
1384#[derive(Clone, Default, Deserialize)]
1385struct RafsConfig {
1386    /// Configuration for storage subsystem.
1387    pub device: FactoryConfig,
1388    /// Filesystem working mode.
1389    pub mode: String,
1390    /// Whether to validate data digest before use.
1391    #[serde(default)]
1392    pub digest_validate: bool,
1393    /// Io statistics.
1394    #[serde(default)]
1395    pub iostats_files: bool,
1396    /// Filesystem prefetching configuration.
1397    #[serde(default)]
1398    pub fs_prefetch: FsPrefetchControl,
1399    /// Enable extended attributes.
1400    #[serde(default)]
1401    pub enable_xattr: bool,
1402    /// Record filesystem access pattern.
1403    #[serde(default)]
1404    pub access_pattern: bool,
1405    /// Record file name if file access trace log.
1406    #[serde(default)]
1407    pub latest_read_files: bool,
1408    // Amplified user IO request batch size to read data from remote storage backend / local cache.
1409    // ZERO value means, amplifying user io is not enabled.
1410    #[serde(rename = "amplify_io", default = "default_user_io_batch_size")]
1411    pub user_io_batch_size: usize,
1412}
1413
1414impl TryFrom<RafsConfig> for ConfigV2 {
1415    type Error = std::io::Error;
1416
1417    fn try_from(v: RafsConfig) -> std::result::Result<Self, Self::Error> {
1418        let backend: BackendConfigV2 = (&v.device.backend).try_into()?;
1419        let mut cache: CacheConfigV2 = (&v.device.cache).try_into()?;
1420        let rafs = RafsConfigV2 {
1421            mode: v.mode,
1422            user_io_batch_size: v.user_io_batch_size,
1423            validate: v.digest_validate,
1424            enable_xattr: v.enable_xattr,
1425            iostats_files: v.iostats_files,
1426            access_pattern: v.access_pattern,
1427            latest_read_files: v.latest_read_files,
1428            prefetch: v.fs_prefetch.into(),
1429        };
1430        if !cache.prefetch.enable && rafs.prefetch.enable {
1431            cache.prefetch = rafs.prefetch.clone();
1432        }
1433
1434        Ok(ConfigV2 {
1435            version: 2,
1436            id: v.device.id,
1437            backend: Some(backend),
1438            external_backends: v.device.external_backends,
1439            cache: Some(cache),
1440            rafs: Some(rafs),
1441            overlay: None,
1442            internal: ConfigV2Internal::default(),
1443        })
1444    }
1445}
1446
1447/// Configuration information for filesystem data prefetch.
1448#[derive(Clone, Default, Deserialize)]
1449struct FsPrefetchControl {
1450    /// Whether the filesystem layer data prefetch is enabled or not.
1451    #[serde(default)]
1452    pub enable: bool,
1453
1454    /// How many working threads to prefetch data.
1455    #[serde(default = "default_prefetch_threads_count")]
1456    pub threads_count: usize,
1457
1458    /// The amplify batch size to prefetch data from backend.
1459    #[serde(rename = "merging_size", default = "default_prefetch_batch_size")]
1460    pub batch_size: usize,
1461
1462    /// Network bandwidth limitation for prefetching.
1463    ///
1464    /// In unit of Bytes. It sets a limit to prefetch bandwidth usage in order to
1465    /// reduce congestion with normal user IO.
1466    /// bandwidth_limit == 0 -- prefetch bandwidth ratelimit disabled
1467    /// bandwidth_limit > 0  -- prefetch bandwidth ratelimit enabled.
1468    ///                        Please note that if the value is less than Rafs chunk size,
1469    ///                        it will be raised to the chunk size.
1470    #[serde(default, rename = "bandwidth_rate")]
1471    pub bandwidth_limit: u32,
1472
1473    /// Whether to prefetch all filesystem data.
1474    #[serde(default = "default_prefetch_all")]
1475    pub prefetch_all: bool,
1476}
1477
1478impl From<FsPrefetchControl> for PrefetchConfigV2 {
1479    fn from(v: FsPrefetchControl) -> Self {
1480        PrefetchConfigV2 {
1481            enable: v.enable,
1482            threads_count: v.threads_count,
1483            batch_size: v.batch_size,
1484            bandwidth_limit: v.bandwidth_limit,
1485            prefetch_all: v.prefetch_all,
1486        }
1487    }
1488}
1489
1490/// Configuration information for blob data prefetching.
1491#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
1492struct BlobPrefetchConfig {
1493    /// Whether to enable blob data prefetching.
1494    pub enable: bool,
1495    /// Number of data prefetching working threads.
1496    pub threads_count: usize,
1497    /// The amplify batch size to prefetch data from backend.
1498    #[serde(rename = "merging_size")]
1499    pub batch_size: usize,
1500    /// Network bandwidth rate limit in unit of Bytes and Zero means no limit.
1501    #[serde(rename = "bandwidth_rate")]
1502    pub bandwidth_limit: u32,
1503}
1504
1505impl From<&BlobPrefetchConfig> for PrefetchConfigV2 {
1506    fn from(v: &BlobPrefetchConfig) -> Self {
1507        PrefetchConfigV2 {
1508            enable: v.enable,
1509            threads_count: v.threads_count,
1510            batch_size: v.batch_size,
1511            bandwidth_limit: v.bandwidth_limit,
1512            prefetch_all: true,
1513        }
1514    }
1515}
1516
1517/// Configuration information for a cached blob, corresponding to `FactoryConfig`.
1518#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1519pub(crate) struct BlobCacheEntryConfig {
1520    /// Identifier for the blob cache configuration: corresponding to `FactoryConfig::id`.
1521    #[serde(default)]
1522    id: String,
1523    /// Type of storage backend, corresponding to `FactoryConfig::BackendConfig::backend_type`.
1524    backend_type: String,
1525    /// Configuration for storage backend, corresponding to `FactoryConfig::BackendConfig::backend_config`.
1526    ///
1527    /// Possible value: `LocalFsConfig`, `RegistryConfig`, `OssConfig`, `LocalDiskConfig`.
1528    backend_config: Value,
1529    /// Configuration for external storage backends, order insensitivity.
1530    #[serde(default)]
1531    external_backends: Vec<ExternalBackendConfig>,
1532    /// Type of blob cache, corresponding to `FactoryConfig::CacheConfig::cache_type`.
1533    ///
1534    /// Possible value: "fscache", "filecache".
1535    cache_type: String,
1536    /// Configuration for blob cache, corresponding to `FactoryConfig::CacheConfig::cache_config`.
1537    ///
1538    /// Possible value: `FileCacheConfig`, `FsCacheConfig`.
1539    cache_config: Value,
1540    /// Configuration for data prefetch.
1541    #[serde(default)]
1542    prefetch_config: BlobPrefetchConfig,
1543    /// Optional file path for metadata blobs.
1544    #[serde(default)]
1545    metadata_path: Option<String>,
1546}
1547
1548impl TryFrom<&BlobCacheEntryConfig> for BlobCacheEntryConfigV2 {
1549    type Error = std::io::Error;
1550
1551    fn try_from(v: &BlobCacheEntryConfig) -> std::result::Result<Self, Self::Error> {
1552        let backend_config = BackendConfig {
1553            backend_type: v.backend_type.clone(),
1554            backend_config: v.backend_config.clone(),
1555        };
1556        let cache_config = CacheConfig {
1557            cache_type: v.cache_type.clone(),
1558            cache_compressed: false,
1559            cache_config: v.cache_config.clone(),
1560            cache_validate: false,
1561            prefetch_config: v.prefetch_config.clone(),
1562        };
1563        Ok(BlobCacheEntryConfigV2 {
1564            version: 2,
1565            id: v.id.clone(),
1566            backend: (&backend_config).try_into()?,
1567            external_backends: v.external_backends.clone(),
1568            cache: (&cache_config).try_into()?,
1569            metadata_path: v.metadata_path.clone(),
1570        })
1571    }
1572}
1573
1574/// Configuration information for Overlay filesystem.
1575/// OverlayConfig is used to configure the writable layer(upper layer),
1576/// The filesystem will be writable when OverlayConfig is set.
1577#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1578pub struct OverlayConfig {
1579    pub upper_dir: String,
1580    pub work_dir: String,
1581}
1582
1583#[cfg(test)]
1584mod tests {
1585    use super::*;
1586    use crate::{BlobCacheEntry, BLOB_CACHE_TYPE_META_BLOB};
1587
1588    #[test]
1589    fn test_blob_prefetch_config() {
1590        let config = BlobPrefetchConfig::default();
1591        assert!(!config.enable);
1592        assert_eq!(config.threads_count, 0);
1593        assert_eq!(config.batch_size, 0);
1594        assert_eq!(config.bandwidth_limit, 0);
1595
1596        let content = r#"{
1597            "enable": true,
1598            "threads_count": 2,
1599            "merging_size": 4,
1600            "bandwidth_rate": 5
1601        }"#;
1602        let config: BlobPrefetchConfig = serde_json::from_str(content).unwrap();
1603        assert!(config.enable);
1604        assert_eq!(config.threads_count, 2);
1605        assert_eq!(config.batch_size, 4);
1606        assert_eq!(config.bandwidth_limit, 5);
1607
1608        let config: PrefetchConfigV2 = (&config).into();
1609        assert!(config.enable);
1610        assert_eq!(config.threads_count, 2);
1611        assert_eq!(config.batch_size, 4);
1612        assert_eq!(config.bandwidth_limit, 5);
1613        assert!(config.prefetch_all);
1614    }
1615
1616    #[test]
1617    fn test_file_cache_config() {
1618        let config: FileCacheConfig = serde_json::from_str("{}").unwrap();
1619        assert_eq!(&config.work_dir, ".");
1620        assert!(!config.disable_indexed_map);
1621
1622        let config: FileCacheConfig =
1623            serde_json::from_str("{\"work_dir\":\"/tmp\",\"disable_indexed_map\":true}").unwrap();
1624        assert_eq!(&config.work_dir, "/tmp");
1625        assert!(config.get_work_dir().is_ok());
1626        assert!(config.disable_indexed_map);
1627
1628        let config: FileCacheConfig =
1629            serde_json::from_str("{\"work_dir\":\"/proc/mounts\",\"disable_indexed_map\":true}")
1630                .unwrap();
1631        assert!(config.get_work_dir().is_err());
1632    }
1633
1634    #[test]
1635    fn test_fs_cache_config() {
1636        let config: FsCacheConfig = serde_json::from_str("{}").unwrap();
1637        assert_eq!(&config.work_dir, ".");
1638
1639        let config: FileCacheConfig = serde_json::from_str("{\"work_dir\":\"/tmp\"}").unwrap();
1640        assert_eq!(&config.work_dir, "/tmp");
1641        assert!(config.get_work_dir().is_ok());
1642
1643        let config: FileCacheConfig =
1644            serde_json::from_str("{\"work_dir\":\"/proc/mounts\"}").unwrap();
1645        assert!(config.get_work_dir().is_err());
1646    }
1647
1648    #[test]
1649    fn test_blob_cache_entry() {
1650        let content = r#"{
1651            "type": "bootstrap",
1652            "id": "blob1",
1653            "config": {
1654                "id": "cache1",
1655                "backend_type": "localfs",
1656                "backend_config": {},
1657                "cache_type": "fscache",
1658                "cache_config": {},
1659                "prefetch_config": {
1660                    "enable": true,
1661                    "threads_count": 2,
1662                    "merging_size": 4,
1663                    "bandwidth_rate": 5
1664                },
1665                "metadata_path": "/tmp/metadata1"
1666            },
1667            "domain_id": "domain1"
1668        }"#;
1669        let config: BlobCacheEntry = serde_json::from_str(content).unwrap();
1670        assert_eq!(&config.blob_type, BLOB_CACHE_TYPE_META_BLOB);
1671        assert_eq!(&config.blob_id, "blob1");
1672        assert_eq!(&config.domain_id, "domain1");
1673
1674        let blob_config = config.blob_config_legacy.as_ref().unwrap();
1675        assert_eq!(blob_config.id, "cache1");
1676        assert_eq!(blob_config.backend_type, "localfs");
1677        assert_eq!(blob_config.cache_type, "fscache");
1678        assert!(blob_config.cache_config.is_object());
1679        assert!(blob_config.prefetch_config.enable);
1680        assert_eq!(blob_config.prefetch_config.threads_count, 2);
1681        assert_eq!(blob_config.prefetch_config.batch_size, 4);
1682        assert_eq!(
1683            blob_config.metadata_path.as_ref().unwrap().as_str(),
1684            "/tmp/metadata1"
1685        );
1686
1687        let blob_config: BlobCacheEntryConfigV2 = blob_config.try_into().unwrap();
1688        assert_eq!(blob_config.id, "cache1");
1689        assert_eq!(blob_config.backend.backend_type, "localfs");
1690        assert_eq!(blob_config.cache.cache_type, "fscache");
1691        assert!(blob_config.cache.fs_cache.is_some());
1692        assert!(blob_config.cache.prefetch.enable);
1693        assert_eq!(blob_config.cache.prefetch.threads_count, 2);
1694        assert_eq!(blob_config.cache.prefetch.batch_size, 4);
1695        assert_eq!(
1696            blob_config.metadata_path.as_ref().unwrap().as_str(),
1697            "/tmp/metadata1"
1698        );
1699
1700        let content = r#"{
1701            "type": "bootstrap",
1702            "id": "blob1",
1703            "config": {
1704                "id": "cache1",
1705                "backend_type": "localfs",
1706                "backend_config": {},
1707                "cache_type": "fscache",
1708                "cache_config": {},
1709                "metadata_path": "/tmp/metadata1"
1710            },
1711            "domain_id": "domain1"
1712        }"#;
1713        let config: BlobCacheEntry = serde_json::from_str(content).unwrap();
1714        let blob_config = config.blob_config_legacy.as_ref().unwrap();
1715        assert!(!blob_config.prefetch_config.enable);
1716        assert_eq!(blob_config.prefetch_config.threads_count, 0);
1717        assert_eq!(blob_config.prefetch_config.batch_size, 0);
1718    }
1719
1720    #[test]
1721    fn test_proxy_config() {
1722        let content = r#"{
1723            "url": "foo.com",
1724            "ping_url": "ping.foo.com",
1725            "fallback": true
1726        }"#;
1727        let config: ProxyConfig = serde_json::from_str(content).unwrap();
1728        assert_eq!(config.url, "foo.com");
1729        assert_eq!(config.ping_url, "ping.foo.com");
1730        assert!(config.fallback);
1731        assert_eq!(config.check_interval, 5);
1732    }
1733
1734    #[test]
1735    fn test_oss_config() {
1736        let content = r#"{
1737            "endpoint": "test",
1738            "access_key_id": "test",
1739            "access_key_secret": "test",
1740            "bucket_name": "antsys-nydus",
1741            "object_prefix":"nydus_v2/"
1742        }"#;
1743        let config: OssConfig = serde_json::from_str(content).unwrap();
1744        assert_eq!(config.scheme, "https");
1745        assert!(!config.skip_verify);
1746        assert_eq!(config.timeout, 5);
1747        assert_eq!(config.connect_timeout, 5);
1748    }
1749
1750    #[test]
1751    fn test_s3_config() {
1752        let content = r#"{
1753            "endpoint": "test",
1754            "region": "us-east-1",
1755            "access_key_id": "test",
1756            "access_key_secret": "test",
1757            "bucket_name": "antsys-nydus",
1758            "object_prefix":"nydus_v2/"
1759        }"#;
1760        let config: OssConfig = serde_json::from_str(content).unwrap();
1761        assert_eq!(config.scheme, "https");
1762        assert!(!config.skip_verify);
1763        assert_eq!(config.timeout, 5);
1764        assert_eq!(config.connect_timeout, 5);
1765    }
1766
1767    #[test]
1768    fn test_registry_config() {
1769        let content = r#"{
1770	    "scheme": "http",
1771            "skip_verify": true,
1772	    "host": "my-registry:5000",
1773	    "repo": "test/repo",
1774	    "auth": "base64_encoded_auth",
1775	    "registry_token": "bearer_token",
1776	    "blob_redirected_host": "blob_redirected_host"
1777        }"#;
1778        let config: RegistryConfig = serde_json::from_str(content).unwrap();
1779        assert_eq!(config.scheme, "http");
1780        assert!(config.skip_verify);
1781    }
1782
1783    #[test]
1784    fn test_localfs_config() {
1785        let content = r#"{
1786            "blob_file": "blob_file",
1787            "dir": "blob_dir",
1788            "alt_dirs": ["dir1", "dir2"]
1789        }"#;
1790        let config: LocalFsConfig = serde_json::from_str(content).unwrap();
1791        assert_eq!(config.blob_file, "blob_file");
1792        assert_eq!(config.dir, "blob_dir");
1793        assert_eq!(config.alt_dirs, vec!["dir1", "dir2"]);
1794    }
1795
1796    #[test]
1797    fn test_localdisk_config() {
1798        let content = r#"{
1799            "device_path": "device_path"
1800        }"#;
1801        let config: LocalDiskConfig = serde_json::from_str(content).unwrap();
1802        assert_eq!(config.device_path, "device_path");
1803    }
1804
1805    #[test]
1806    fn test_backend_config() {
1807        let config = BackendConfig {
1808            backend_type: "localfs".to_string(),
1809            backend_config: Default::default(),
1810        };
1811        let str_val = serde_json::to_string(&config).unwrap();
1812        let config2 = serde_json::from_str(&str_val).unwrap();
1813
1814        assert_eq!(config, config2);
1815    }
1816
1817    #[test]
1818    fn test_v2_version() {
1819        let content = "version=2";
1820        let config: ConfigV2 = toml::from_str(content).unwrap();
1821        assert_eq!(config.version, 2);
1822        assert!(config.backend.is_none());
1823    }
1824
1825    #[test]
1826    fn test_v2_backend() {
1827        let content = r#"version=2
1828        [backend]
1829        type = "localfs"
1830        "#;
1831        let config: ConfigV2 = toml::from_str(content).unwrap();
1832        assert_eq!(config.version, 2);
1833        assert!(config.backend.is_some());
1834        assert!(config.cache.is_none());
1835
1836        let backend = config.backend.as_ref().unwrap();
1837        assert_eq!(&backend.backend_type, "localfs");
1838        assert!(backend.localfs.is_none());
1839        assert!(backend.oss.is_none());
1840        assert!(backend.registry.is_none());
1841    }
1842
1843    #[test]
1844    fn test_v2_backend_localfs() {
1845        let content = r#"version=2
1846        [backend]
1847        type = "localfs"
1848        [backend.localfs]
1849        blob_file = "/tmp/nydus.blob.data"
1850        dir = "/tmp"
1851        alt_dirs = ["/var/nydus/cache"]
1852        "#;
1853        let config: ConfigV2 = toml::from_str(content).unwrap();
1854        assert_eq!(config.version, 2);
1855        assert!(config.backend.is_some());
1856
1857        let backend = config.backend.as_ref().unwrap();
1858        assert_eq!(&backend.backend_type, "localfs");
1859        assert!(backend.localfs.is_some());
1860
1861        let localfs = backend.localfs.as_ref().unwrap();
1862        assert_eq!(&localfs.blob_file, "/tmp/nydus.blob.data");
1863        assert_eq!(&localfs.dir, "/tmp");
1864        assert_eq!(&localfs.alt_dirs[0], "/var/nydus/cache");
1865    }
1866
1867    #[test]
1868    fn test_v2_backend_oss() {
1869        let content = r#"version=2
1870        [backend]
1871        type = "oss"
1872        [backend.oss]
1873        endpoint = "my_endpoint"
1874        bucket_name = "my_bucket_name"
1875        object_prefix = "my_object_prefix"
1876        access_key_id = "my_access_key_id"
1877        access_key_secret = "my_access_key_secret"
1878        scheme = "http"
1879        skip_verify = true
1880        timeout = 10
1881        connect_timeout = 10
1882        retry_limit = 5
1883        [backend.oss.proxy]
1884        url = "localhost:6789"
1885        ping_url = "localhost:6789/ping"
1886        fallback = true
1887        check_interval = 10
1888        use_http = true
1889        [[backend.oss.mirrors]]
1890        host = "http://127.0.0.1:65001"
1891        ping_url = "http://127.0.0.1:65001/ping"
1892        health_check_interval = 10
1893        failure_limit = 10
1894        "#;
1895        let config: ConfigV2 = toml::from_str(content).unwrap();
1896        assert_eq!(config.version, 2);
1897        assert!(config.backend.is_some());
1898        assert!(config.rafs.is_none());
1899
1900        let backend = config.backend.as_ref().unwrap();
1901        assert_eq!(&backend.backend_type, "oss");
1902        assert!(backend.oss.is_some());
1903
1904        let oss = backend.oss.as_ref().unwrap();
1905        assert_eq!(&oss.endpoint, "my_endpoint");
1906        assert_eq!(&oss.bucket_name, "my_bucket_name");
1907        assert_eq!(&oss.object_prefix, "my_object_prefix");
1908        assert_eq!(&oss.access_key_id, "my_access_key_id");
1909        assert_eq!(&oss.access_key_secret, "my_access_key_secret");
1910        assert_eq!(&oss.scheme, "http");
1911        assert!(oss.skip_verify);
1912        assert_eq!(oss.timeout, 10);
1913        assert_eq!(oss.connect_timeout, 10);
1914        assert_eq!(oss.retry_limit, 5);
1915        assert_eq!(&oss.proxy.url, "localhost:6789");
1916        assert_eq!(&oss.proxy.ping_url, "localhost:6789/ping");
1917        assert_eq!(oss.proxy.check_interval, 10);
1918        assert!(oss.proxy.fallback);
1919        assert!(oss.proxy.use_http);
1920
1921        assert_eq!(oss.mirrors.len(), 1);
1922        let mirror = &oss.mirrors[0];
1923        assert_eq!(mirror.host, "http://127.0.0.1:65001");
1924        assert_eq!(mirror.ping_url, "http://127.0.0.1:65001/ping");
1925        assert!(mirror.headers.is_empty());
1926        assert_eq!(mirror.health_check_interval, 10);
1927        assert_eq!(mirror.failure_limit, 10);
1928    }
1929
1930    #[test]
1931    fn test_v2_backend_registry() {
1932        let content = r#"version=2
1933        [backend]
1934        type = "registry"
1935        [backend.registry]
1936        scheme = "http"
1937        host = "localhost"
1938        repo = "nydus"
1939        auth = "auth"
1940        skip_verify = true
1941        timeout = 10
1942        connect_timeout = 10
1943        retry_limit = 5
1944        registry_token = "bear_token"
1945        blob_url_scheme = "https"
1946        blob_redirected_host = "redirect.registry.com"
1947        [backend.registry.proxy]
1948        url = "localhost:6789"
1949        ping_url = "localhost:6789/ping"
1950        fallback = true
1951        check_interval = 10
1952        use_http = true
1953        [[backend.registry.mirrors]]
1954        host = "http://127.0.0.1:65001"
1955        ping_url = "http://127.0.0.1:65001/ping"
1956        health_check_interval = 10
1957        failure_limit = 10
1958        "#;
1959        let config: ConfigV2 = toml::from_str(content).unwrap();
1960        assert_eq!(config.version, 2);
1961        assert!(config.backend.is_some());
1962        assert!(config.rafs.is_none());
1963
1964        let backend = config.backend.as_ref().unwrap();
1965        assert_eq!(&backend.backend_type, "registry");
1966        assert!(backend.registry.is_some());
1967
1968        let registry = backend.registry.as_ref().unwrap();
1969        assert_eq!(&registry.scheme, "http");
1970        assert_eq!(&registry.host, "localhost");
1971        assert_eq!(&registry.repo, "nydus");
1972        assert_eq!(registry.auth.as_ref().unwrap(), "auth");
1973        assert!(registry.skip_verify);
1974        assert_eq!(registry.timeout, 10);
1975        assert_eq!(registry.connect_timeout, 10);
1976        assert_eq!(registry.retry_limit, 5);
1977        assert_eq!(registry.registry_token.as_ref().unwrap(), "bear_token");
1978        assert_eq!(registry.blob_url_scheme, "https");
1979        assert_eq!(registry.blob_redirected_host, "redirect.registry.com");
1980
1981        assert_eq!(&registry.proxy.url, "localhost:6789");
1982        assert_eq!(&registry.proxy.ping_url, "localhost:6789/ping");
1983        assert_eq!(registry.proxy.check_interval, 10);
1984        assert!(registry.proxy.fallback);
1985        assert!(registry.proxy.use_http);
1986
1987        assert_eq!(registry.mirrors.len(), 1);
1988        let mirror = &registry.mirrors[0];
1989        assert_eq!(mirror.host, "http://127.0.0.1:65001");
1990        assert_eq!(mirror.ping_url, "http://127.0.0.1:65001/ping");
1991        assert!(mirror.headers.is_empty());
1992        assert_eq!(mirror.health_check_interval, 10);
1993        assert_eq!(mirror.failure_limit, 10);
1994    }
1995
1996    #[test]
1997    fn test_v2_cache() {
1998        let content = r#"version=2
1999        [cache]
2000        type = "filecache"
2001        compressed = true
2002        validate = true
2003        [cache.filecache]
2004        work_dir = "/tmp"
2005        [cache.fscache]
2006        work_dir = "./"
2007        [cache.prefetch]
2008        enable = true
2009        threads = 8
2010        batch_size = 1000000
2011        bandwidth_limit = 10000000
2012        "#;
2013        let config: ConfigV2 = toml::from_str(content).unwrap();
2014        assert_eq!(config.version, 2);
2015        assert!(config.backend.is_none());
2016        assert!(config.rafs.is_none());
2017        assert!(config.cache.is_some());
2018
2019        let cache = config.cache.as_ref().unwrap();
2020        assert_eq!(&cache.cache_type, "filecache");
2021        assert!(cache.cache_compressed);
2022        assert!(cache.cache_validate);
2023        let filecache = cache.file_cache.as_ref().unwrap();
2024        assert_eq!(&filecache.work_dir, "/tmp");
2025        let fscache = cache.fs_cache.as_ref().unwrap();
2026        assert_eq!(&fscache.work_dir, "./");
2027
2028        let prefetch = &cache.prefetch;
2029        assert!(prefetch.enable);
2030        assert_eq!(prefetch.threads_count, 8);
2031        assert_eq!(prefetch.batch_size, 1000000);
2032        assert_eq!(prefetch.bandwidth_limit, 10000000);
2033    }
2034
2035    #[test]
2036    fn test_v2_rafs() {
2037        let content = r#"version=2
2038        [rafs]
2039        mode = "direct"
2040        batch_size = 1000000
2041        validate = true
2042        enable_xattr = true
2043        iostats_files = true
2044        access_pattern = true
2045        latest_read_files = true
2046        [rafs.prefetch]
2047        enable = true
2048        threads = 4
2049        batch_size = 1000000
2050        bandwidth_limit = 10000000
2051        prefetch_all = true
2052        "#;
2053        let config: ConfigV2 = toml::from_str(content).unwrap();
2054        assert_eq!(config.version, 2);
2055        assert!(config.backend.is_none());
2056        assert!(config.cache.is_none());
2057        assert!(config.rafs.is_some());
2058
2059        let rafs = config.rafs.as_ref().unwrap();
2060        assert_eq!(&rafs.mode, "direct");
2061        assert_eq!(rafs.user_io_batch_size, 1000000);
2062        assert!(rafs.validate);
2063        assert!(rafs.enable_xattr);
2064        assert!(rafs.iostats_files);
2065        assert!(rafs.access_pattern);
2066        assert!(rafs.latest_read_files);
2067        assert!(rafs.prefetch.enable);
2068        assert_eq!(rafs.prefetch.threads_count, 4);
2069        assert_eq!(rafs.prefetch.batch_size, 1000000);
2070        assert_eq!(rafs.prefetch.bandwidth_limit, 10000000);
2071        assert!(rafs.prefetch.prefetch_all)
2072    }
2073
2074    #[test]
2075    fn test_v2_blob_cache_entry() {
2076        let content = r#"version=2
2077        id = "my_id"
2078        metadata_path = "meta_path"
2079        [backend]
2080        type = "localfs"
2081        [backend.localfs]
2082        blob_file = "/tmp/nydus.blob.data"
2083        dir = "/tmp"
2084        alt_dirs = ["/var/nydus/cache"]
2085        [cache]
2086        type = "filecache"
2087        compressed = true
2088        validate = true
2089        [cache.filecache]
2090        work_dir = "/tmp"
2091        "#;
2092        let config: BlobCacheEntryConfigV2 = toml::from_str(content).unwrap();
2093        assert_eq!(config.version, 2);
2094        assert_eq!(&config.id, "my_id");
2095        assert_eq!(config.metadata_path.as_ref().unwrap(), "meta_path");
2096
2097        let backend = &config.backend;
2098        assert_eq!(&backend.backend_type, "localfs");
2099        assert!(backend.localfs.is_some());
2100
2101        let localfs = backend.localfs.as_ref().unwrap();
2102        assert_eq!(&localfs.blob_file, "/tmp/nydus.blob.data");
2103        assert_eq!(&localfs.dir, "/tmp");
2104        assert_eq!(&localfs.alt_dirs[0], "/var/nydus/cache");
2105    }
2106
2107    #[test]
2108    fn test_sample_config_file() {
2109        let content = r#"{
2110            "device": {
2111                "backend": {
2112                    "type": "localfs",
2113                    "config": {
2114                        "dir": "/tmp/AM7TxD/blobs",
2115                        "readahead": true
2116                    }
2117                },
2118                "cache": {
2119                    "type": "blobcache",
2120                    "compressed": true,
2121                    "config": {
2122                        "work_dir": "/tmp/AM7TxD/cache"
2123                    }
2124                }
2125            },
2126            "mode": "cached",
2127            "digest_validate": true,
2128            "iostats_files": false
2129        }
2130        "#;
2131        let config = ConfigV2::from_str(content).unwrap();
2132        assert_eq!(&config.id, "");
2133    }
2134
2135    #[test]
2136    fn test_snapshotter_sample_config() {
2137        let content = r#"
2138        {
2139            "device": {
2140                "backend": {
2141                    "type": "registry",
2142                    "config": {
2143                        "readahead": false,
2144                        "host": "localhost",
2145                        "repo": "vke/golang",
2146                        "auth": "",
2147                        "scheme": "https",
2148                        "proxy": {
2149                            "fallback": false
2150                        },
2151                        "timeout": 5,
2152                        "connect_timeout": 5,
2153                        "retry_limit": 2
2154                    }
2155                },
2156                "cache": {
2157                    "type": "blobcache",
2158                    "compressed": true,
2159                    "config": {
2160                        "work_dir": "/var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache",
2161                        "disable_indexed_map": false
2162                    }
2163                }
2164            },
2165            "mode": "direct",
2166            "digest_validate": false,
2167            "enable_xattr": true,
2168            "fs_prefetch": {
2169                "enable": true,
2170                "prefetch_all": true,
2171                "threads_count": 8,
2172                "merging_size": 1048576,
2173                "bandwidth_rate": 0
2174            }
2175        }
2176        "#;
2177        let config = ConfigV2::from_str(content).unwrap();
2178        assert_eq!(&config.id, "");
2179    }
2180
2181    #[test]
2182    fn test_backend_http_proxy_config() {
2183        let config =
2184            r#"{"version":2,"backend":{"type":"http-proxy","http-proxy":{"addr":"/tmp"}}}"#;
2185        let config = ConfigV2::from_str(config).unwrap();
2186        let backend = config.backend.unwrap();
2187        assert_eq!(&backend.backend_type, "http-proxy");
2188        assert_eq!(&backend.http_proxy.unwrap().addr, "/tmp");
2189    }
2190
2191    #[test]
2192    fn test_new_localfs() {
2193        let config = ConfigV2::new_localfs("id1", "./").unwrap();
2194        assert_eq!(&config.id, "id1");
2195        assert_eq!(config.backend.as_ref().unwrap().backend_type, "localfs");
2196    }
2197
2198    #[test]
2199    fn test_update_registry_auth_info() {
2200        let config = r#"
2201        {
2202            "device": {
2203              "id": "test",
2204              "backend": {
2205                "type": "registry",
2206                "config": {
2207                    "readahead": false,
2208                    "host": "docker.io",
2209                    "repo": "library/nginx",
2210                    "scheme": "https",
2211                    "proxy": {
2212                        "fallback": false
2213                    },
2214                    "timeout": 5,
2215                    "connect_timeout": 5,
2216                    "retry_limit": 8
2217                }
2218              }
2219            },
2220            "mode": "direct",
2221            "digest_validate": false,
2222            "enable_xattr": true,
2223            "fs_prefetch": {
2224              "enable": true,
2225              "threads_count": 10,
2226              "merging_size": 131072,
2227              "bandwidth_rate": 10485760
2228            }
2229          }"#;
2230
2231        let mut rafs_config = ConfigV2::from_str(&config).unwrap();
2232        let test_auth = "test_auth".to_string();
2233
2234        rafs_config.update_registry_auth_info(&Some(test_auth.clone()));
2235
2236        let backend = rafs_config.backend.unwrap();
2237        let registry = backend.registry.unwrap();
2238        let auth = registry.auth.unwrap();
2239        assert_eq!(auth, test_auth);
2240    }
2241
2242    #[test]
2243    fn test_config2_error() {
2244        let content_bad_version = r#"version=3
2245        "#;
2246        let cfg: ConfigV2 = toml::from_str(content_bad_version).unwrap();
2247        assert!(!cfg.validate());
2248        let cfg = ConfigV2::new("id");
2249        assert!(cfg.get_backend_config().is_err());
2250        assert!(cfg.get_cache_config().is_err());
2251        assert!(cfg.get_rafs_config().is_err());
2252        assert!(cfg.get_cache_working_directory().is_err());
2253
2254        let content = r#"version=2
2255            [cache]
2256            type = "filecache"
2257            [cache.filecache]
2258            work_dir = "/tmp"
2259        "#;
2260        let cfg: ConfigV2 = toml::from_str(content).unwrap();
2261        assert_eq!(cfg.get_cache_working_directory().unwrap(), "/tmp");
2262
2263        let content = r#"version=2
2264            [cache]
2265            type = "fscache"
2266            [cache.fscache]
2267            work_dir = "./foo"
2268        "#;
2269        let cfg: ConfigV2 = toml::from_str(content).unwrap();
2270        assert_eq!(cfg.get_cache_working_directory().unwrap(), "./foo");
2271
2272        let content = r#"version=2
2273            [cache]
2274            type = "bar"
2275        "#;
2276        let cfg: ConfigV2 = toml::from_str(content).unwrap();
2277        assert!(cfg.get_cache_working_directory().is_err());
2278
2279        let content = r#"
2280            foo-bar-xxxx
2281        "#;
2282        assert!(toml::from_str::<ConfigV2>(content).is_err());
2283    }
2284
2285    #[test]
2286    fn test_backend_config_valid() {
2287        let mut cfg = BackendConfigV2 {
2288            backend_type: "localdisk".to_string(),
2289            ..Default::default()
2290        };
2291        assert!(!cfg.validate());
2292        cfg.localdisk = Some(LocalDiskConfig {
2293            device_path: "".to_string(),
2294            disable_gpt: true,
2295        });
2296        assert!(!cfg.validate());
2297
2298        let cfg = BackendConfigV2 {
2299            backend_type: "localfs".to_string(),
2300            ..Default::default()
2301        };
2302        assert!(!cfg.validate());
2303
2304        let cfg = BackendConfigV2 {
2305            backend_type: "oss".to_string(),
2306            ..Default::default()
2307        };
2308        assert!(!cfg.validate());
2309
2310        let cfg = BackendConfigV2 {
2311            backend_type: "s3".to_string(),
2312            ..Default::default()
2313        };
2314        assert!(!cfg.validate());
2315
2316        let cfg = BackendConfigV2 {
2317            backend_type: "register".to_string(),
2318            ..Default::default()
2319        };
2320        assert!(!cfg.validate());
2321
2322        let cfg = BackendConfigV2 {
2323            backend_type: "http-proxy".to_string(),
2324            ..Default::default()
2325        };
2326        assert!(!cfg.validate());
2327
2328        let cfg = BackendConfigV2 {
2329            backend_type: "foobar".to_string(),
2330            ..Default::default()
2331        };
2332        assert!(!cfg.validate());
2333    }
2334
2335    fn get_config(backend_type: &str) {
2336        let mut cfg: BackendConfigV2 = BackendConfigV2::default();
2337        assert!(cfg.get_localdisk_config().is_err());
2338
2339        cfg.backend_type = backend_type.to_string();
2340        assert!(cfg.get_localdisk_config().is_err());
2341    }
2342
2343    #[test]
2344    fn test_get_config() {
2345        get_config("localdisk");
2346        get_config("localfs");
2347        get_config("oss");
2348        get_config("s3");
2349        get_config("register");
2350        get_config("http-proxy");
2351    }
2352
2353    #[test]
2354    fn test_cache_config_valid() {
2355        let cfg = CacheConfigV2 {
2356            cache_type: "blobcache".to_string(),
2357            ..Default::default()
2358        };
2359        assert!(!cfg.validate());
2360
2361        let cfg = CacheConfigV2 {
2362            cache_type: "fscache".to_string(),
2363            ..Default::default()
2364        };
2365        assert!(!cfg.validate());
2366
2367        let cfg = CacheConfigV2 {
2368            cache_type: "dummycache".to_string(),
2369            ..Default::default()
2370        };
2371        assert!(cfg.validate());
2372
2373        let cfg = CacheConfigV2 {
2374            cache_type: "foobar".to_string(),
2375            ..Default::default()
2376        };
2377        assert!(!cfg.validate());
2378    }
2379
2380    #[test]
2381    fn test_get_fscache_config() {
2382        let mut cfg = CacheConfigV2::default();
2383        assert!(cfg.get_fscache_config().is_err());
2384        cfg.cache_type = "fscache".to_string();
2385        assert!(cfg.get_fscache_config().is_err());
2386    }
2387
2388    #[test]
2389    fn test_fscache_get_work_dir() {
2390        let mut cfg = FsCacheConfig::default();
2391        assert!(cfg.get_work_dir().is_err());
2392        cfg.work_dir = ".".to_string();
2393        assert!(cfg.get_work_dir().is_ok());
2394        cfg.work_dir = "foobar".to_string();
2395        let res = cfg.get_work_dir().is_ok();
2396        fs::remove_dir_all("foobar").unwrap();
2397        assert!(res);
2398    }
2399
2400    #[test]
2401    fn test_default_mirror_config() {
2402        let cfg = MirrorConfig::default();
2403        assert_eq!(cfg.host, "");
2404        assert_eq!(cfg.health_check_interval, 5);
2405        assert_eq!(cfg.failure_limit, 5);
2406        assert_eq!(cfg.ping_url, "");
2407    }
2408
2409    #[test]
2410    fn test_config_v2_from_file() {
2411        let content = r#"version=2
2412            [cache]
2413            type = "filecache"
2414            [cache.filecache]
2415            work_dir = "/tmp"
2416        "#;
2417        if fs::write("test_config_v2_from_file.cfg", content).is_ok() {
2418            let res = ConfigV2::from_file("test_config_v2_from_file.cfg").is_ok();
2419            fs::remove_file("test_config_v2_from_file.cfg").unwrap();
2420            assert!(res);
2421        } else {
2422            assert!(ConfigV2::from_file("test_config_v2_from_file.cfg").is_err());
2423        }
2424    }
2425
2426    #[test]
2427    fn test_blob_cache_entry_v2_from_file() {
2428        let content = r#"version=2
2429        id = "my_id"
2430        metadata_path = "meta_path"
2431        [backend]
2432        type = "localfs"
2433        [backend.localfs]
2434        blob_file = "/tmp/nydus.blob.data"
2435        dir = "/tmp"
2436        alt_dirs = ["/var/nydus/cache"]
2437        [cache]
2438        type = "filecache"
2439        compressed = true
2440        validate = true
2441        [cache.filecache]
2442        work_dir = "/tmp"
2443        "#;
2444        if fs::write("test_blob_cache_entry_v2_from_file.cfg", content).is_ok() {
2445            let res =
2446                BlobCacheEntryConfigV2::from_file("test_blob_cache_entry_v2_from_file.cfg").is_ok();
2447            fs::remove_file("test_blob_cache_entry_v2_from_file.cfg").unwrap();
2448            assert!(res);
2449        } else {
2450            assert!(ConfigV2::from_file("test_blob_cache_entry_v2_from_file.cfg").is_err());
2451        }
2452    }
2453
2454    #[test]
2455    fn test_blob_cache_valid() {
2456        let err_version_content = r#"version=1"#;
2457
2458        let config: BlobCacheEntryConfigV2 = toml::from_str(err_version_content).unwrap();
2459        assert!(!config.validate());
2460
2461        let content = r#"version=2
2462        id = "my_id"
2463        metadata_path = "meta_path"
2464        [backend]
2465        type = "localfs"
2466        [backend.localfs]
2467        blob_file = "/tmp/nydus.blob.data"
2468        dir = "/tmp"
2469        alt_dirs = ["/var/nydus/cache"]
2470        [cache]
2471        type = "filecache"
2472        compressed = true
2473        validate = true
2474        [cache.filecache]
2475        work_dir = "/tmp"
2476        "#;
2477
2478        let config: BlobCacheEntryConfigV2 = toml::from_str(content).unwrap();
2479        assert!(config.validate());
2480    }
2481
2482    #[test]
2483    fn test_blob_from_str() {
2484        let content = r#"version=2
2485        id = "my_id"
2486        metadata_path = "meta_path"
2487        [backend]
2488        type = "localfs"
2489        [backend.localfs]
2490        blob_file = "/tmp/nydus.blob.data"
2491        dir = "/tmp"
2492        alt_dirs = ["/var/nydus/cache"]
2493        [cache]
2494        type = "filecache"
2495        compressed = true
2496        validate = true
2497        [cache.filecache]
2498        work_dir = "/tmp"
2499        "#;
2500
2501        let config: BlobCacheEntryConfigV2 = BlobCacheEntryConfigV2::from_str(content).unwrap();
2502        assert_eq!(config.version, 2);
2503        assert_eq!(config.id, "my_id");
2504        assert_eq!(config.backend.localfs.unwrap().dir, "/tmp");
2505        assert_eq!(config.cache.file_cache.unwrap().work_dir, "/tmp");
2506        let content = r#"
2507            {
2508                "version": 2,
2509                "id": "my_id",
2510                "backend": {
2511                    "type": "localfs",
2512                    "localfs": {
2513                        "dir": "/tmp"
2514                    }
2515                }
2516            }
2517        "#;
2518        let config: BlobCacheEntryConfigV2 = BlobCacheEntryConfigV2::from_str(content).unwrap();
2519
2520        assert_eq!(config.version, 2);
2521        assert_eq!(config.id, "my_id");
2522        assert_eq!(config.backend.localfs.unwrap().dir, "/tmp");
2523
2524        let content = r#"foobar"#;
2525        assert!(BlobCacheEntryConfigV2::from_str(content).is_err());
2526    }
2527
2528    #[test]
2529    fn test_blob_cache_entry_from_file() {
2530        let content = r#"{
2531            "type": "bootstrap",
2532            "id": "blob1",
2533            "config": {
2534                "id": "cache1",
2535                "backend_type": "localfs",
2536                "backend_config": {},
2537                "cache_type": "fscache",
2538                "cache_config": {},
2539                "metadata_path": "/tmp/metadata1"
2540            },
2541            "domain_id": "domain1"
2542        }"#;
2543        if fs::write("test_blob_cache_entry_from_file.cfg", content).is_ok() {
2544            let res = BlobCacheEntry::from_file("test_blob_cache_entry_from_file.cfg").is_ok();
2545            fs::remove_file("test_blob_cache_entry_from_file.cfg").unwrap();
2546            assert!(res);
2547        } else {
2548            assert!(ConfigV2::from_file("test_blob_cache_entry_from_file.cfg").is_err());
2549        }
2550    }
2551
2552    #[test]
2553    fn test_blob_cache_entry_valid() {
2554        let content = r#"{
2555            "type": "bootstrap",
2556            "id": "blob1",
2557            "config": {
2558                "id": "cache1",
2559                "backend_type": "localfs",
2560                "backend_config": {},
2561                "cache_type": "fscache",
2562                "cache_config": {},
2563                "metadata_path": "/tmp/metadata1"
2564            },
2565            "domain_id": "domain1"
2566        }"#;
2567        let mut cfg = BlobCacheEntry::from_str(content).unwrap();
2568        cfg.blob_type = "foobar".to_string();
2569        assert!(!cfg.validate());
2570
2571        let content = r#"{
2572            "type": "bootstrap",
2573            "id": "blob1",
2574            "domain_id": "domain1"
2575        }"#;
2576        let cfg = BlobCacheEntry::from_str(content).unwrap();
2577        assert!(cfg.validate());
2578    }
2579
2580    #[test]
2581    fn test_blob_cache_entry_from_str() {
2582        let content = r#"{
2583            "type": "bootstrap",
2584            "id": "blob1",
2585            "config": {
2586                "id": "cache1",
2587                "backend_type": "localfs",
2588                "backend_config": {},
2589                "cache_type": "fscache",
2590                "cache_config": {},
2591                "metadata_path": "/tmp/metadata1"
2592            },
2593            "domain_id": "domain1"
2594        }"#;
2595        assert!(BlobCacheEntry::from_str(content).is_ok());
2596        let content = r#"{
2597            "type": "foobar",
2598            "id": "blob1",
2599            "config": {
2600                "id": "cache1",
2601                "backend_type": "foobar",
2602                "backend_config": {},
2603                "cache_type": "foobar",
2604                "cache_config": {},
2605                "metadata_path": "/tmp/metadata1"
2606            },
2607            "domain_id": "domain1"
2608        }"#;
2609        assert!(BlobCacheEntry::from_str(content).is_err());
2610
2611        let content = r#"foobar"#;
2612        assert!(BlobCacheEntry::from_str(content).is_err());
2613    }
2614
2615    #[test]
2616    fn test_default_value() {
2617        assert!(default_true());
2618        assert_eq!(default_failure_limit(), 5);
2619        assert_eq!(default_prefetch_batch_size(), 1024 * 1024);
2620        assert_eq!(default_prefetch_threads_count(), 8);
2621    }
2622
2623    #[test]
2624    fn test_backend_config_try_from() {
2625        let config = BackendConfig {
2626            backend_type: "localdisk".to_string(),
2627            backend_config: serde_json::to_value(LocalDiskConfig::default()).unwrap(),
2628        };
2629        assert!(BackendConfigV2::try_from(&config).is_ok());
2630
2631        let config = BackendConfig {
2632            backend_type: "localfs".to_string(),
2633            backend_config: serde_json::to_value(LocalFsConfig::default()).unwrap(),
2634        };
2635        assert!(BackendConfigV2::try_from(&config).is_ok());
2636
2637        let config = BackendConfig {
2638            backend_type: "oss".to_string(),
2639            backend_config: serde_json::to_value(OssConfig::default()).unwrap(),
2640        };
2641        assert!(BackendConfigV2::try_from(&config).is_ok());
2642
2643        let config = BackendConfig {
2644            backend_type: "s3".to_string(),
2645            backend_config: serde_json::to_value(S3Config::default()).unwrap(),
2646        };
2647        assert!(BackendConfigV2::try_from(&config).is_ok());
2648
2649        let config = BackendConfig {
2650            backend_type: "registry".to_string(),
2651            backend_config: serde_json::to_value(RegistryConfig::default()).unwrap(),
2652        };
2653        assert!(BackendConfigV2::try_from(&config).is_ok());
2654
2655        let config = BackendConfig {
2656            backend_type: "foobar".to_string(),
2657            backend_config: serde_json::to_value(LocalDiskConfig::default()).unwrap(),
2658        };
2659        assert!(BackendConfigV2::try_from(&config).is_err());
2660    }
2661}
nydus_api/config.rs

nydus_api/
config.rs
