nydus_api/
config.rs

1// Copyright 2022 Alibaba Cloud. All rights reserved.
2// Copyright 2020 Ant Group. All rights reserved.
3//
4// SPDX-License-Identifier: Apache-2.0
5
6use std::collections::HashMap;
7use std::convert::{TryFrom, TryInto};
8use std::fs;
9use std::io::{Error, ErrorKind, Result};
10use std::path::Path;
11use std::str::FromStr;
12use std::sync::atomic::{AtomicBool, Ordering};
13use std::sync::Arc;
14
15use serde::Deserialize;
16use serde_json::Value;
17
18/// Configuration file format version 2, based on Toml.
19#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
20pub struct ConfigV2 {
21    /// Configuration file format version number, must be 2.
22    pub version: u32,
23    /// Identifier for the instance.
24    #[serde(default)]
25    pub id: String,
26    /// Configuration information for storage backend.
27    pub backend: Option<BackendConfigV2>,
28    /// Configuration for external storage backends, order insensitivity.
29    #[serde(default)]
30    pub external_backends: Vec<ExternalBackendConfig>,
31    /// Configuration information for local cache system.
32    pub cache: Option<CacheConfigV2>,
33    /// Configuration information for RAFS filesystem.
34    pub rafs: Option<RafsConfigV2>,
35    /// Overlay configuration information for the instance.
36    pub overlay: Option<OverlayConfig>,
37    /// Internal runtime configuration.
38    #[serde(skip)]
39    pub internal: ConfigV2Internal,
40}
41
42impl Default for ConfigV2 {
43    fn default() -> Self {
44        ConfigV2 {
45            version: 2,
46            id: String::new(),
47            backend: None,
48            external_backends: Vec::new(),
49            cache: None,
50            rafs: None,
51            overlay: None,
52            internal: ConfigV2Internal::default(),
53        }
54    }
55}
56
57impl ConfigV2 {
58    /// Create a new instance of `ConfigV2` object.
59    pub fn new(id: &str) -> Self {
60        ConfigV2 {
61            version: 2,
62            id: id.to_string(),
63            backend: None,
64            external_backends: Vec::new(),
65            cache: None,
66            rafs: None,
67            overlay: None,
68            internal: ConfigV2Internal::default(),
69        }
70    }
71
72    /// Create a new configuration object for `backend-localfs` and `filecache`.
73    pub fn new_localfs(id: &str, dir: &str) -> Result<Self> {
74        let content = format!(
75            r#"
76        version = 2
77        id = "{}"
78        backend.type = "localfs"
79        backend.localfs.dir = "{}"
80        cache.type = "filecache"
81        cache.compressed = false
82        cache.validate = false
83        cache.filecache.work_dir = "{}"
84        "#,
85            id, dir, dir
86        );
87
88        Self::from_str(&content)
89    }
90
91    /// Read configuration information from a file.
92    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
93        let md = fs::metadata(path.as_ref())?;
94        if md.len() > 0x100000 {
95            return Err(Error::other("configuration file size is too big"));
96        }
97        let content = fs::read_to_string(path)?;
98        Self::from_str(&content)
99    }
100
101    /// Validate the configuration object.
102    pub fn validate(&self) -> bool {
103        if self.version != 2 {
104            return false;
105        }
106        if let Some(backend_cfg) = self.backend.as_ref() {
107            if !backend_cfg.validate() {
108                return false;
109            }
110        }
111        if let Some(cache_cfg) = self.cache.as_ref() {
112            if !cache_cfg.validate() {
113                return false;
114            }
115        }
116        if let Some(rafs_cfg) = self.rafs.as_ref() {
117            if !rafs_cfg.validate() {
118                return false;
119            }
120        }
121
122        true
123    }
124
125    /// Get configuration information for storage backend.
126    pub fn get_backend_config(&self) -> Result<&BackendConfigV2> {
127        self.backend.as_ref().ok_or_else(|| {
128            Error::new(
129                ErrorKind::InvalidInput,
130                "no configuration information for backend",
131            )
132        })
133    }
134
135    /// Get configuration information for cache subsystem.
136    pub fn get_cache_config(&self) -> Result<&CacheConfigV2> {
137        self.cache.as_ref().ok_or_else(|| {
138            Error::new(
139                ErrorKind::InvalidData,
140                "no configuration information for cache",
141            )
142        })
143    }
144
145    /// Get cache working directory.
146    pub fn get_cache_working_directory(&self) -> Result<String> {
147        let cache = self.get_cache_config()?;
148        if cache.is_filecache() {
149            if let Some(c) = cache.file_cache.as_ref() {
150                return Ok(c.work_dir.clone());
151            }
152        } else if cache.is_fscache() {
153            if let Some(c) = cache.fs_cache.as_ref() {
154                return Ok(c.work_dir.clone());
155            }
156        }
157
158        Err(Error::new(
159            ErrorKind::NotFound,
160            "no working directory configured",
161        ))
162    }
163
164    /// Get configuration information for RAFS filesystem.
165    pub fn get_rafs_config(&self) -> Result<&RafsConfigV2> {
166        self.rafs.as_ref().ok_or_else(|| {
167            Error::new(
168                ErrorKind::InvalidInput,
169                "no configuration information for rafs",
170            )
171        })
172    }
173
174    /// Clone the object with all secrets removed.
175    pub fn clone_without_secrets(&self) -> Self {
176        let mut cfg = self.clone();
177
178        if let Some(backend_cfg) = cfg.backend.as_mut() {
179            if let Some(oss_cfg) = backend_cfg.oss.as_mut() {
180                oss_cfg.access_key_id = String::new();
181                oss_cfg.access_key_secret = String::new();
182            }
183            if let Some(registry_cfg) = backend_cfg.registry.as_mut() {
184                registry_cfg.auth = None;
185                registry_cfg.registry_token = None;
186            }
187        }
188
189        cfg
190    }
191
192    /// Check whether chunk digest validation is enabled or not.
193    pub fn is_chunk_validation_enabled(&self) -> bool {
194        let mut validation = if let Some(cache) = &self.cache {
195            cache.cache_validate
196        } else {
197            false
198        };
199        if let Some(rafs) = &self.rafs {
200            if rafs.validate {
201                validation = true;
202            }
203        }
204
205        validation
206    }
207
208    /// Check whether fscache is enabled or not.
209    pub fn is_fs_cache(&self) -> bool {
210        if let Some(cache) = self.cache.as_ref() {
211            cache.fs_cache.is_some()
212        } else {
213            false
214        }
215    }
216
217    /// Fill authorization for registry backend.
218    pub fn update_registry_auth_info(&mut self, auth: &Option<String>) {
219        if let Some(auth) = auth {
220            if let Some(backend) = self.backend.as_mut() {
221                if let Some(registry) = backend.registry.as_mut() {
222                    registry.auth = Some(auth.to_string());
223                }
224            }
225        }
226    }
227}
228
229impl FromStr for ConfigV2 {
230    type Err = std::io::Error;
231
232    fn from_str(s: &str) -> Result<ConfigV2> {
233        if let Ok(v) = serde_json::from_str::<ConfigV2>(s) {
234            return if v.validate() {
235                Ok(v)
236            } else {
237                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
238            };
239        }
240        if let Ok(v) = toml::from_str::<ConfigV2>(s) {
241            return if v.validate() {
242                Ok(v)
243            } else {
244                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
245            };
246        }
247        if let Ok(v) = serde_json::from_str::<RafsConfig>(s) {
248            if let Ok(v) = ConfigV2::try_from(v) {
249                if v.validate() {
250                    return Ok(v);
251                }
252            }
253        }
254        Err(Error::new(
255            ErrorKind::InvalidInput,
256            "failed to parse configuration information",
257        ))
258    }
259}
260
261/// Configuration information for storage backend.
262#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
263pub struct BackendConfigV2 {
264    /// Type of storage backend.
265    #[serde(rename = "type")]
266    pub backend_type: String,
267    /// Configuration for local disk backend.
268    pub localdisk: Option<LocalDiskConfig>,
269    /// Configuration for local filesystem backend.
270    pub localfs: Option<LocalFsConfig>,
271    /// Configuration for OSS backend.
272    pub oss: Option<OssConfig>,
273    /// Configuration for S3 backend.
274    pub s3: Option<S3Config>,
275    /// Configuration for container registry backend.
276    pub registry: Option<RegistryConfig>,
277    /// Configuration for local http proxy.
278    #[serde(rename = "http-proxy")]
279    pub http_proxy: Option<HttpProxyConfig>,
280}
281
282impl BackendConfigV2 {
283    /// Validate storage backend configuration.
284    pub fn validate(&self) -> bool {
285        match self.backend_type.as_str() {
286            "localdisk" => match self.localdisk.as_ref() {
287                Some(v) => {
288                    if v.device_path.is_empty() {
289                        return false;
290                    }
291                }
292                None => return false,
293            },
294            "localfs" => match self.localfs.as_ref() {
295                Some(v) => {
296                    if v.blob_file.is_empty() && v.dir.is_empty() {
297                        return false;
298                    }
299                }
300                None => return false,
301            },
302            "oss" => match self.oss.as_ref() {
303                Some(v) => {
304                    if v.endpoint.is_empty() || v.bucket_name.is_empty() {
305                        return false;
306                    }
307                }
308                None => return false,
309            },
310            "s3" => match self.s3.as_ref() {
311                Some(v) => {
312                    if v.region.is_empty() || v.bucket_name.is_empty() {
313                        return false;
314                    }
315                }
316                None => return false,
317            },
318            "registry" => match self.registry.as_ref() {
319                Some(v) => {
320                    if v.host.is_empty() || v.repo.is_empty() {
321                        return false;
322                    }
323                }
324                None => return false,
325            },
326
327            "http-proxy" => match self.http_proxy.as_ref() {
328                Some(v) => {
329                    let is_valid_unix_socket_path = |path: &str| {
330                        let path = Path::new(path);
331                        path.is_absolute() && path.exists()
332                    };
333                    if v.addr.is_empty()
334                        || !(v.addr.starts_with("http://")
335                            || v.addr.starts_with("https://")
336                            || is_valid_unix_socket_path(&v.addr))
337                    {
338                        return false;
339                    }
340
341                    // check if v.path is valid url path format
342                    if Path::new(&v.path).join("any_blob_id").to_str().is_none() {
343                        return false;
344                    }
345                }
346                None => return false,
347            },
348            _ => return false,
349        }
350
351        true
352    }
353
354    /// Get configuration information for localdisk
355    pub fn get_localdisk_config(&self) -> Result<&LocalDiskConfig> {
356        if &self.backend_type != "localdisk" {
357            Err(Error::new(
358                ErrorKind::InvalidInput,
359                "backend type is not 'localdisk'",
360            ))
361        } else {
362            self.localdisk.as_ref().ok_or_else(|| {
363                Error::new(
364                    ErrorKind::InvalidData,
365                    "no configuration information for localdisk",
366                )
367            })
368        }
369    }
370
371    /// Get configuration information for localfs
372    pub fn get_localfs_config(&self) -> Result<&LocalFsConfig> {
373        if &self.backend_type != "localfs" {
374            Err(Error::new(
375                ErrorKind::InvalidInput,
376                "backend type is not 'localfs'",
377            ))
378        } else {
379            self.localfs.as_ref().ok_or_else(|| {
380                Error::new(
381                    ErrorKind::InvalidData,
382                    "no configuration information for localfs",
383                )
384            })
385        }
386    }
387
388    /// Get configuration information for OSS
389    pub fn get_oss_config(&self) -> Result<&OssConfig> {
390        if &self.backend_type != "oss" {
391            Err(Error::new(
392                ErrorKind::InvalidInput,
393                "backend type is not 'oss'",
394            ))
395        } else {
396            self.oss.as_ref().ok_or_else(|| {
397                Error::new(
398                    ErrorKind::InvalidData,
399                    "no configuration information for OSS",
400                )
401            })
402        }
403    }
404
405    /// Get configuration information for S3
406    pub fn get_s3_config(&self) -> Result<&S3Config> {
407        if &self.backend_type != "s3" {
408            Err(Error::new(
409                ErrorKind::InvalidInput,
410                "backend type is not 's3'",
411            ))
412        } else {
413            self.s3.as_ref().ok_or_else(|| {
414                Error::new(
415                    ErrorKind::InvalidData,
416                    "no configuration information for s3",
417                )
418            })
419        }
420    }
421
422    /// Get configuration information for Registry
423    pub fn get_registry_config(&self) -> Result<&RegistryConfig> {
424        if &self.backend_type != "registry" {
425            Err(Error::new(
426                ErrorKind::InvalidInput,
427                "backend type is not 'registry'",
428            ))
429        } else {
430            self.registry.as_ref().ok_or_else(|| {
431                Error::new(
432                    ErrorKind::InvalidData,
433                    "no configuration information for registry",
434                )
435            })
436        }
437    }
438
439    /// Get configuration information for http proxy
440    pub fn get_http_proxy_config(&self) -> Result<&HttpProxyConfig> {
441        if &self.backend_type != "http-proxy" {
442            Err(Error::new(
443                ErrorKind::InvalidInput,
444                "backend type is not 'http-proxy'",
445            ))
446        } else {
447            self.http_proxy.as_ref().ok_or_else(|| {
448                Error::new(
449                    ErrorKind::InvalidData,
450                    "no configuration information for http-proxy",
451                )
452            })
453        }
454    }
455}
456
457/// Configuration information for localdisk storage backend.
458#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
459pub struct LocalDiskConfig {
460    /// Mounted block device path or original localdisk image file path.
461    #[serde(default)]
462    pub device_path: String,
463    /// Disable discover blob objects by scanning GPT table.
464    #[serde(default)]
465    pub disable_gpt: bool,
466}
467
468/// Configuration information for localfs storage backend.
469#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
470pub struct LocalFsConfig {
471    /// Blob file to access.
472    #[serde(default)]
473    pub blob_file: String,
474    /// Dir to hold blob files. Used when 'blob_file' is not specified.
475    #[serde(default)]
476    pub dir: String,
477    /// Alternative dirs to search for blobs.
478    #[serde(default)]
479    pub alt_dirs: Vec<String>,
480}
481
482/// OSS configuration information to access blobs.
483#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
484pub struct OssConfig {
485    /// Oss http scheme, either 'http' or 'https'
486    #[serde(default = "default_http_scheme")]
487    pub scheme: String,
488    /// Oss endpoint
489    pub endpoint: String,
490    /// Oss bucket name
491    pub bucket_name: String,
492    /// Prefix object_prefix to OSS object key, for example the simulation of subdirectory:
493    /// - object_key: sha256:xxx
494    /// - object_prefix: nydus/
495    /// - object_key with object_prefix: nydus/sha256:xxx
496    #[serde(default)]
497    pub object_prefix: String,
498    /// Oss access key
499    #[serde(default)]
500    pub access_key_id: String,
501    /// Oss secret
502    #[serde(default)]
503    pub access_key_secret: String,
504    /// Skip SSL certificate validation for HTTPS scheme.
505    #[serde(default)]
506    pub skip_verify: bool,
507    /// Drop the read request once http request timeout, in seconds.
508    #[serde(default = "default_http_timeout")]
509    pub timeout: u32,
510    /// Drop the read request once http connection timeout, in seconds.
511    #[serde(default = "default_http_timeout")]
512    pub connect_timeout: u32,
513    /// Retry count when read request failed.
514    #[serde(default)]
515    pub retry_limit: u8,
516    /// Enable HTTP proxy for the read request.
517    #[serde(default)]
518    pub proxy: ProxyConfig,
519}
520
521/// S3 configuration information to access blobs.
522#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
523pub struct S3Config {
524    /// S3 http scheme, either 'http' or 'https'
525    #[serde(default = "default_http_scheme")]
526    pub scheme: String,
527    /// S3 endpoint
528    pub endpoint: String,
529    /// S3 region
530    pub region: String,
531    /// S3 bucket name
532    pub bucket_name: String,
533    /// Prefix object_prefix to S3 object key, for example the simulation of subdirectory:
534    /// - object_key: sha256:xxx
535    /// - object_prefix: nydus/
536    /// - object_key with object_prefix: nydus/sha256:xxx
537    #[serde(default)]
538    pub object_prefix: String,
539    /// S3 access key
540    #[serde(default)]
541    pub access_key_id: String,
542    /// S3 secret
543    #[serde(default)]
544    pub access_key_secret: String,
545    /// Skip SSL certificate validation for HTTPS scheme.
546    #[serde(default)]
547    pub skip_verify: bool,
548    /// Drop the read request once http request timeout, in seconds.
549    #[serde(default = "default_http_timeout")]
550    pub timeout: u32,
551    /// Drop the read request once http connection timeout, in seconds.
552    #[serde(default = "default_http_timeout")]
553    pub connect_timeout: u32,
554    /// Retry count when read request failed.
555    #[serde(default)]
556    pub retry_limit: u8,
557    /// Enable HTTP proxy for the read request.
558    #[serde(default)]
559    pub proxy: ProxyConfig,
560}
561
562/// Http proxy configuration information to access blobs.
563#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
564pub struct HttpProxyConfig {
565    /// Address of http proxy server, like `http://xxx.xxx` or `https://xxx.xxx` or `/path/to/unix.sock`.
566    pub addr: String,
567    /// Path to access the blobs, like `/<_namespace>/<_repo>/blobs`.
568    /// If the http proxy server is over unix socket, this field will be ignored.
569    #[serde(default)]
570    pub path: String,
571    /// Skip SSL certificate validation for HTTPS scheme.
572    #[serde(default)]
573    pub skip_verify: bool,
574    /// Drop the read request once http request timeout, in seconds.
575    #[serde(default = "default_http_timeout")]
576    pub timeout: u32,
577    /// Drop the read request once http connection timeout, in seconds.
578    #[serde(default = "default_http_timeout")]
579    pub connect_timeout: u32,
580    /// Retry count when read request failed.
581    #[serde(default)]
582    pub retry_limit: u8,
583    /// Enable HTTP proxy for the read request.
584    #[serde(default)]
585    pub proxy: ProxyConfig,
586}
587
588/// Container registry configuration information to access blobs.
589#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
590pub struct RegistryConfig {
591    /// Registry http scheme, either 'http' or 'https'
592    #[serde(default = "default_http_scheme")]
593    pub scheme: String,
594    /// Registry url host
595    pub host: String,
596    /// Registry image name, like 'library/ubuntu'
597    pub repo: String,
598    /// Base64_encoded(username:password), the field should be sent to registry auth server to get a bearer token.
599    #[serde(default)]
600    pub auth: Option<String>,
601    /// Skip SSL certificate validation for HTTPS scheme.
602    #[serde(default)]
603    pub skip_verify: bool,
604    /// Drop the read request once http request timeout, in seconds.
605    #[serde(default = "default_http_timeout")]
606    pub timeout: u32,
607    /// Drop the read request once http connection timeout, in seconds.
608    #[serde(default = "default_http_timeout")]
609    pub connect_timeout: u32,
610    /// Retry count when read request failed.
611    #[serde(default)]
612    pub retry_limit: u8,
613    /// The field is a bearer token to be sent to registry to authorize registry requests.
614    #[serde(default)]
615    pub registry_token: Option<String>,
616    /// The http scheme to access blobs. It is used to workaround some P2P subsystem
617    /// that requires a different scheme than the registry.
618    #[serde(default)]
619    pub blob_url_scheme: String,
620    /// Redirect blob access to a different host regardless of the one specified in 'host'.
621    #[serde(default)]
622    pub blob_redirected_host: String,
623    /// Enable HTTP proxy for the read request.
624    #[serde(default)]
625    pub proxy: ProxyConfig,
626    /// Disable background token refresh thread. Defaults to false.
627    #[serde(skip_deserializing)]
628    pub disable_token_refresh: bool,
629}
630
631/// Configuration information for blob cache manager.
632#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
633pub struct CacheConfigV2 {
634    /// Type of blob cache: "blobcache", "fscache" or "dummy"
635    #[serde(default, rename = "type")]
636    pub cache_type: String,
637    /// Whether the data from the cache is compressed, not used anymore.
638    #[serde(default, rename = "compressed")]
639    pub cache_compressed: bool,
640    /// Whether to validate data read from the cache.
641    #[serde(default, rename = "validate")]
642    pub cache_validate: bool,
643    /// Configuration for blob level prefetch.
644    #[serde(default)]
645    pub prefetch: PrefetchConfigV2,
646    /// Configuration information for file cache
647    #[serde(rename = "filecache")]
648    pub file_cache: Option<FileCacheConfig>,
649    #[serde(rename = "fscache")]
650    /// Configuration information for fscache
651    pub fs_cache: Option<FsCacheConfig>,
652}
653
654impl CacheConfigV2 {
655    /// Validate cache configuration information.
656    pub fn validate(&self) -> bool {
657        match self.cache_type.as_str() {
658            "blobcache" | "filecache" => {
659                if let Some(c) = self.file_cache.as_ref() {
660                    if c.work_dir.is_empty() {
661                        return false;
662                    }
663                } else {
664                    return false;
665                }
666            }
667            "fscache" => {
668                if let Some(c) = self.fs_cache.as_ref() {
669                    if c.work_dir.is_empty() {
670                        return false;
671                    }
672                } else {
673                    return false;
674                }
675            }
676            "" | "dummycache" => {}
677            _ => return false,
678        }
679
680        if self.prefetch.enable {
681            if self.prefetch.batch_size > 0x10000000 {
682                return false;
683            }
684            if self.prefetch.threads_count == 0 || self.prefetch.threads_count > 1024 {
685                return false;
686            }
687        }
688
689        true
690    }
691
692    /// Check whether the cache type is `filecache`
693    pub fn is_filecache(&self) -> bool {
694        self.cache_type == "blobcache" || self.cache_type == "filecache"
695    }
696
697    /// Check whether the cache type is `fscache`
698    pub fn is_fscache(&self) -> bool {
699        self.cache_type == "fscache"
700    }
701
702    /// Get configuration information for file cache.
703    pub fn get_filecache_config(&self) -> Result<&FileCacheConfig> {
704        if self.is_filecache() {
705            self.file_cache.as_ref().ok_or_else(|| {
706                Error::new(
707                    ErrorKind::InvalidInput,
708                    "no configuration information for filecache",
709                )
710            })
711        } else {
712            Err(Error::new(
713                ErrorKind::InvalidData,
714                "cache type is not 'filecache'",
715            ))
716        }
717    }
718
719    /// Get configuration information for fscache.
720    pub fn get_fscache_config(&self) -> Result<&FsCacheConfig> {
721        if self.is_fscache() {
722            self.fs_cache.as_ref().ok_or_else(|| {
723                Error::new(
724                    ErrorKind::InvalidData,
725                    "no configuration information for fscache",
726                )
727            })
728        } else {
729            Err(Error::new(
730                ErrorKind::InvalidInput,
731                "cache type is not 'fscache'",
732            ))
733        }
734    }
735}
736
737/// Configuration information for file cache.
738#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
739pub struct FileCacheConfig {
740    /// Working directory to store state and cached files.
741    #[serde(default = "default_work_dir")]
742    pub work_dir: String,
743    /// Deprecated: disable index mapping, keep it as false when possible.
744    #[serde(default)]
745    pub disable_indexed_map: bool,
746    /// Enable encryption data written to the cache file.
747    #[serde(default)]
748    pub enable_encryption: bool,
749    /// Enable convergent encryption for chunk deduplication.
750    #[serde(default)]
751    pub enable_convergent_encryption: bool,
752    /// Key for data encryption, a heximal representation of [u8; 32].
753    #[serde(default)]
754    pub encryption_key: String,
755}
756
757impl FileCacheConfig {
758    /// Get the working directory.
759    pub fn get_work_dir(&self) -> Result<&str> {
760        let path = fs::metadata(&self.work_dir)
761            .or_else(|_| {
762                fs::create_dir_all(&self.work_dir)?;
763                fs::metadata(&self.work_dir)
764            })
765            .map_err(|e| {
766                log::error!("fail to stat filecache work_dir {}: {}", self.work_dir, e);
767                e
768            })?;
769
770        if path.is_dir() {
771            Ok(&self.work_dir)
772        } else {
773            Err(Error::new(
774                ErrorKind::NotFound,
775                format!("filecache work_dir {} is not a directory", self.work_dir),
776            ))
777        }
778    }
779}
780
781/// Configuration information for fscache.
782#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
783pub struct FsCacheConfig {
784    /// Working directory to store state and cached files.
785    #[serde(default = "default_work_dir")]
786    pub work_dir: String,
787}
788
789impl FsCacheConfig {
790    /// Get the working directory.
791    pub fn get_work_dir(&self) -> Result<&str> {
792        let path = fs::metadata(&self.work_dir)
793            .or_else(|_| {
794                fs::create_dir_all(&self.work_dir)?;
795                fs::metadata(&self.work_dir)
796            })
797            .map_err(|e| {
798                log::error!("fail to stat fscache work_dir {}: {}", self.work_dir, e);
799                e
800            })?;
801
802        if path.is_dir() {
803            Ok(&self.work_dir)
804        } else {
805            Err(Error::new(
806                ErrorKind::NotFound,
807                format!("fscache work_dir {} is not a directory", self.work_dir),
808            ))
809        }
810    }
811}
812
813/// Configuration information for RAFS filesystem.
814#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
815pub struct RafsConfigV2 {
816    /// Filesystem metadata cache mode.
817    #[serde(default = "default_rafs_mode")]
818    pub mode: String,
819    /// Amplified user IO request batch size to read data from remote storage backend / local cache.
820    #[serde(rename = "batch_size", default = "default_user_io_batch_size")]
821    pub user_io_batch_size: usize,
822    /// Whether to validate data digest.
823    #[serde(default)]
824    pub validate: bool,
825    /// Enable support of extended attributes.
826    #[serde(default)]
827    pub enable_xattr: bool,
828    /// Record file operation metrics for each file.
829    ///
830    /// Better to keep it off in production environment due to possible resource consumption.
831    #[serde(default)]
832    pub iostats_files: bool,
833    /// Record filesystem access pattern.
834    #[serde(default)]
835    pub access_pattern: bool,
836    /// Record file name if file access trace log.
837    #[serde(default)]
838    pub latest_read_files: bool,
839    /// Filesystem prefetching configuration.
840    #[serde(default)]
841    pub prefetch: PrefetchConfigV2,
842}
843
844impl RafsConfigV2 {
845    /// Validate RAFS filesystem configuration information.
846    pub fn validate(&self) -> bool {
847        if self.mode != "direct" && self.mode != "cached" {
848            return false;
849        }
850        if self.user_io_batch_size > 0x10000000 {
851            return false;
852        }
853        if self.prefetch.enable {
854            if self.prefetch.batch_size > 0x10000000 {
855                return false;
856            }
857            if self.prefetch.threads_count == 0 || self.prefetch.threads_count > 1024 {
858                return false;
859            }
860        }
861
862        true
863    }
864}
865
866/// Configuration information for blob data prefetching.
867#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
868pub struct PrefetchConfigV2 {
869    /// Whether to enable blob data prefetching.
870    pub enable: bool,
871    /// Number of data prefetching working threads.
872    #[serde(rename = "threads", default = "default_prefetch_threads_count")]
873    pub threads_count: usize,
874    /// The amplify batch size to prefetch data from backend.
875    #[serde(default = "default_prefetch_batch_size")]
876    pub batch_size: usize,
877    /// Network bandwidth rate limit in unit of Bytes and Zero means no limit.
878    #[serde(default)]
879    pub bandwidth_limit: u32,
880    /// Prefetch all data from backend.
881    #[serde(default)]
882    pub prefetch_all: bool,
883}
884
885/// Configuration information for network proxy.
886#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
887pub struct ProxyConfig {
888    /// Access remote storage backend via proxy, e.g. Dragonfly dfdaemon server URL.
889    #[serde(default)]
890    pub url: String,
891    /// Proxy health checking endpoint.
892    #[serde(default)]
893    pub ping_url: String,
894    /// Fallback to remote storage backend if proxy ping failed.
895    #[serde(default = "default_true")]
896    pub fallback: bool,
897    /// Interval for proxy health checking, in seconds.
898    #[serde(default = "default_check_interval")]
899    pub check_interval: u64,
900    /// Replace URL to http to request source registry with proxy, and allow fallback to https if the proxy is unhealthy.
901    #[serde(default)]
902    pub use_http: bool,
903    /// Elapsed time to pause proxy health check when the request is inactive, in seconds.
904    #[serde(default = "default_check_pause_elapsed")]
905    pub check_pause_elapsed: u64,
906}
907
908impl Default for ProxyConfig {
909    fn default() -> Self {
910        Self {
911            url: String::new(),
912            ping_url: String::new(),
913            fallback: true,
914            check_interval: 5,
915            use_http: false,
916            check_pause_elapsed: 300,
917        }
918    }
919}
920
921/// Configuration information for a cached blob`.
922#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
923pub struct BlobCacheEntryConfigV2 {
924    /// Configuration file format version number, must be 2.
925    pub version: u32,
926    /// Identifier for the instance.
927    #[serde(default)]
928    pub id: String,
929    /// Configuration information for storage backend.
930    #[serde(default)]
931    pub backend: BackendConfigV2,
932    /// Configuration for external storage backends, order insensitivity.
933    #[serde(default)]
934    pub external_backends: Vec<ExternalBackendConfig>,
935    /// Configuration information for local cache system.
936    #[serde(default)]
937    pub cache: CacheConfigV2,
938    /// Optional file path for metadata blob.
939    #[serde(default)]
940    pub metadata_path: Option<String>,
941}
942
943impl BlobCacheEntryConfigV2 {
944    /// Read configuration information from a file.
945    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
946        let md = fs::metadata(path.as_ref())?;
947        if md.len() > 0x100000 {
948            return Err(Error::new(
949                ErrorKind::InvalidInput,
950                "configuration file size is too big",
951            ));
952        }
953        let content = fs::read_to_string(path)?;
954        Self::from_str(&content)
955    }
956
957    /// Validate the configuration object.
958    pub fn validate(&self) -> bool {
959        if self.version != 2 {
960            return false;
961        }
962        let config: ConfigV2 = self.into();
963        config.validate()
964    }
965}
966
967impl FromStr for BlobCacheEntryConfigV2 {
968    type Err = Error;
969
970    fn from_str(s: &str) -> Result<BlobCacheEntryConfigV2> {
971        if let Ok(v) = serde_json::from_str::<BlobCacheEntryConfigV2>(s) {
972            return if v.validate() {
973                Ok(v)
974            } else {
975                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
976            };
977        }
978        if let Ok(v) = toml::from_str::<BlobCacheEntryConfigV2>(s) {
979            return if v.validate() {
980                Ok(v)
981            } else {
982                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
983            };
984        }
985        Err(Error::new(
986            ErrorKind::InvalidInput,
987            "failed to parse configuration information",
988        ))
989    }
990}
991
992impl From<&BlobCacheEntryConfigV2> for ConfigV2 {
993    fn from(c: &BlobCacheEntryConfigV2) -> Self {
994        ConfigV2 {
995            version: c.version,
996            id: c.id.clone(),
997            backend: Some(c.backend.clone()),
998            external_backends: c.external_backends.clone(),
999            cache: Some(c.cache.clone()),
1000            rafs: None,
1001            overlay: None,
1002            internal: ConfigV2Internal::default(),
1003        }
1004    }
1005}
1006
1007/// Internal runtime configuration.
1008#[derive(Clone, Debug)]
1009pub struct ConfigV2Internal {
1010    /// It's possible to access the raw or more blob objects.
1011    pub blob_accessible: Arc<AtomicBool>,
1012}
1013
1014impl Default for ConfigV2Internal {
1015    fn default() -> Self {
1016        ConfigV2Internal {
1017            blob_accessible: Arc::new(AtomicBool::new(false)),
1018        }
1019    }
1020}
1021
1022impl PartialEq for ConfigV2Internal {
1023    fn eq(&self, other: &Self) -> bool {
1024        self.blob_accessible() == other.blob_accessible()
1025    }
1026}
1027
1028impl Eq for ConfigV2Internal {}
1029
1030impl ConfigV2Internal {
1031    /// Get the auto-probe flag.
1032    pub fn blob_accessible(&self) -> bool {
1033        self.blob_accessible.load(Ordering::Relaxed)
1034    }
1035
1036    /// Set the auto-probe flag.
1037    pub fn set_blob_accessible(&self, accessible: bool) {
1038        self.blob_accessible.store(accessible, Ordering::Relaxed);
1039    }
1040}
1041
1042/// Blob cache object type for nydus/rafs bootstrap blob.
1043pub const BLOB_CACHE_TYPE_META_BLOB: &str = "bootstrap";
1044/// Blob cache object type for nydus/rafs data blob.
1045pub const BLOB_CACHE_TYPE_DATA_BLOB: &str = "datablob";
1046
1047/// Configuration information for a cached blob.
1048#[derive(Debug, Deserialize, Serialize, Clone)]
1049pub struct BlobCacheEntry {
1050    /// Type of blob object, bootstrap or data blob.
1051    #[serde(rename = "type")]
1052    pub blob_type: String,
1053    /// Blob id.
1054    #[serde(rename = "id")]
1055    pub blob_id: String,
1056    /// Configuration information to generate blob cache object.
1057    #[serde(default, rename = "config")]
1058    pub(crate) blob_config_legacy: Option<BlobCacheEntryConfig>,
1059    /// Configuration information to generate blob cache object.
1060    #[serde(default, rename = "config_v2")]
1061    pub blob_config: Option<BlobCacheEntryConfigV2>,
1062    /// Domain id for the blob, which is used to group cached blobs into management domains.
1063    #[serde(default)]
1064    pub domain_id: String,
1065}
1066
1067impl BlobCacheEntry {
1068    pub fn prepare_configuration_info(&mut self) -> bool {
1069        if self.blob_config.is_none() {
1070            if let Some(legacy) = self.blob_config_legacy.as_ref() {
1071                match legacy.try_into() {
1072                    Err(_) => return false,
1073                    Ok(v) => self.blob_config = Some(v),
1074                }
1075            }
1076        }
1077
1078        match self.blob_config.as_ref() {
1079            None => false,
1080            Some(cfg) => cfg.cache.validate() && cfg.backend.validate(),
1081        }
1082    }
1083}
1084
1085impl BlobCacheEntry {
1086    /// Read configuration information from a file.
1087    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
1088        let md = fs::metadata(path.as_ref())?;
1089        if md.len() > 0x100000 {
1090            return Err(Error::new(
1091                ErrorKind::InvalidInput,
1092                "configuration file size is too big",
1093            ));
1094        }
1095        let content = fs::read_to_string(path)?;
1096        Self::from_str(&content)
1097    }
1098
1099    /// Validate the configuration object.
1100    pub fn validate(&self) -> bool {
1101        if self.blob_type != BLOB_CACHE_TYPE_META_BLOB
1102            && self.blob_type != BLOB_CACHE_TYPE_DATA_BLOB
1103        {
1104            log::warn!("invalid blob type {} for blob cache entry", self.blob_type);
1105            return false;
1106        }
1107        if let Some(config) = self.blob_config.as_ref() {
1108            if !config.validate() {
1109                return false;
1110            }
1111        }
1112        true
1113    }
1114}
1115
1116impl FromStr for BlobCacheEntry {
1117    type Err = Error;
1118
1119    fn from_str(s: &str) -> Result<BlobCacheEntry> {
1120        if let Ok(v) = serde_json::from_str::<BlobCacheEntry>(s) {
1121            return if v.validate() {
1122                Ok(v)
1123            } else {
1124                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
1125            };
1126        }
1127        if let Ok(v) = toml::from_str::<BlobCacheEntry>(s) {
1128            return if v.validate() {
1129                Ok(v)
1130            } else {
1131                Err(Error::new(ErrorKind::InvalidInput, "invalid configuration"))
1132            };
1133        }
1134        Err(Error::new(
1135            ErrorKind::InvalidInput,
1136            "failed to parse configuration information",
1137        ))
1138    }
1139}
1140
1141/// Configuration information for a list of cached blob objects.
1142#[derive(Debug, Default, Deserialize, Serialize)]
1143pub struct BlobCacheList {
1144    /// List of blob configuration information.
1145    pub blobs: Vec<BlobCacheEntry>,
1146}
1147
1148fn default_true() -> bool {
1149    true
1150}
1151
1152fn default_http_scheme() -> String {
1153    "https".to_string()
1154}
1155
1156fn default_http_timeout() -> u32 {
1157    5
1158}
1159
1160fn default_check_interval() -> u64 {
1161    5
1162}
1163
1164fn default_check_pause_elapsed() -> u64 {
1165    300
1166}
1167
1168fn default_work_dir() -> String {
1169    ".".to_string()
1170}
1171
1172pub fn default_user_io_batch_size() -> usize {
1173    1024 * 1024
1174}
1175
1176pub fn default_prefetch_batch_size() -> usize {
1177    1024 * 1024
1178}
1179
1180fn default_prefetch_threads_count() -> usize {
1181    8
1182}
1183
1184fn default_prefetch_all() -> bool {
1185    true
1186}
1187
1188fn default_rafs_mode() -> String {
1189    "direct".to_string()
1190}
1191
1192////////////////////////////////////////////////////////////////////////////////////////////////////
1193// For backward compatibility
1194////////////////////////////////////////////////////////////////////////////////////////////////////
1195
1196/// Configuration information for storage backend.
1197#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1198struct BackendConfig {
1199    /// Type of storage backend.
1200    #[serde(rename = "type")]
1201    pub backend_type: String,
1202    /// Configuration for storage backend.
1203    /// Possible value: `LocalFsConfig`, `RegistryConfig`, `OssConfig`, `LocalDiskConfig`.
1204    #[serde(rename = "config")]
1205    pub backend_config: Value,
1206}
1207
1208impl TryFrom<&BackendConfig> for BackendConfigV2 {
1209    type Error = std::io::Error;
1210
1211    fn try_from(value: &BackendConfig) -> std::result::Result<Self, Self::Error> {
1212        let mut config = BackendConfigV2 {
1213            backend_type: value.backend_type.clone(),
1214            localdisk: None,
1215            localfs: None,
1216            oss: None,
1217            s3: None,
1218            registry: None,
1219            http_proxy: None,
1220        };
1221
1222        match value.backend_type.as_str() {
1223            "localdisk" => {
1224                config.localdisk = Some(serde_json::from_value(value.backend_config.clone())?);
1225            }
1226            "localfs" => {
1227                config.localfs = Some(serde_json::from_value(value.backend_config.clone())?);
1228            }
1229            "oss" => {
1230                config.oss = Some(serde_json::from_value(value.backend_config.clone())?);
1231            }
1232            "s3" => {
1233                config.s3 = Some(serde_json::from_value(value.backend_config.clone())?);
1234            }
1235            "registry" => {
1236                config.registry = Some(serde_json::from_value(value.backend_config.clone())?);
1237            }
1238            v => {
1239                return Err(Error::new(
1240                    ErrorKind::InvalidInput,
1241                    format!("unsupported backend type '{}'", v),
1242                ))
1243            }
1244        }
1245
1246        Ok(config)
1247    }
1248}
1249
1250/// Configuration information for blob cache manager.
1251#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1252struct CacheConfig {
1253    /// Type of blob cache: "blobcache", "fscache" or ""
1254    #[serde(default, rename = "type")]
1255    pub cache_type: String,
1256    /// Whether the data from the cache is compressed, not used anymore.
1257    #[serde(default, rename = "compressed")]
1258    pub cache_compressed: bool,
1259    /// Blob cache manager specific configuration: FileCacheConfig, FsCacheConfig.
1260    #[serde(default, rename = "config")]
1261    pub cache_config: Value,
1262    /// Whether to validate data read from the cache.
1263    #[serde(default, rename = "validate")]
1264    pub cache_validate: bool,
1265    /// Configuration for blob data prefetching.
1266    #[serde(skip_serializing, skip_deserializing)]
1267    pub prefetch_config: BlobPrefetchConfig,
1268}
1269
1270/// Additional configuration information for external backend, its items
1271/// will be merged to the configuration from image.
1272#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1273pub struct ExternalBackendConfig {
1274    /// External backend identifier to merge.
1275    pub patch: HashMap<String, String>,
1276    /// External backend type.
1277    #[serde(rename = "type")]
1278    pub kind: String,
1279    /// External backend config items to merge.
1280    pub config: HashMap<String, String>,
1281}
1282
1283impl TryFrom<&CacheConfig> for CacheConfigV2 {
1284    type Error = std::io::Error;
1285
1286    fn try_from(v: &CacheConfig) -> std::result::Result<Self, Self::Error> {
1287        let mut config = CacheConfigV2 {
1288            cache_type: v.cache_type.clone(),
1289            cache_compressed: v.cache_compressed,
1290            cache_validate: v.cache_validate,
1291            prefetch: (&v.prefetch_config).into(),
1292            file_cache: None,
1293            fs_cache: None,
1294        };
1295
1296        match v.cache_type.as_str() {
1297            "blobcache" | "filecache" => {
1298                config.file_cache = Some(serde_json::from_value(v.cache_config.clone())?);
1299            }
1300            "fscache" => {
1301                config.fs_cache = Some(serde_json::from_value(v.cache_config.clone())?);
1302            }
1303            "" | "dummycache" => {}
1304            t => {
1305                return Err(Error::new(
1306                    ErrorKind::InvalidInput,
1307                    format!("unsupported cache type '{}'", t),
1308                ))
1309            }
1310        }
1311
1312        Ok(config)
1313    }
1314}
1315
1316/// Configuration information to create blob cache manager.
1317#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1318struct FactoryConfig {
1319    /// Id of the factory.
1320    #[serde(default)]
1321    pub id: String,
1322    /// Configuration for storage backend.
1323    pub backend: BackendConfig,
1324    /// Configuration for external storage backends, order insensitivity.
1325    #[serde(default)]
1326    pub external_backends: Vec<ExternalBackendConfig>,
1327    /// Configuration for blob cache manager.
1328    #[serde(default)]
1329    pub cache: CacheConfig,
1330}
1331
1332/// Rafs storage backend configuration information.
1333#[derive(Clone, Default, Deserialize)]
1334struct RafsConfig {
1335    /// Configuration for storage subsystem.
1336    pub device: FactoryConfig,
1337    /// Filesystem working mode.
1338    pub mode: String,
1339    /// Whether to validate data digest before use.
1340    #[serde(default)]
1341    pub digest_validate: bool,
1342    /// Io statistics.
1343    #[serde(default)]
1344    pub iostats_files: bool,
1345    /// Filesystem prefetching configuration.
1346    #[serde(default)]
1347    pub fs_prefetch: FsPrefetchControl,
1348    /// Enable extended attributes.
1349    #[serde(default)]
1350    pub enable_xattr: bool,
1351    /// Record filesystem access pattern.
1352    #[serde(default)]
1353    pub access_pattern: bool,
1354    /// Record file name if file access trace log.
1355    #[serde(default)]
1356    pub latest_read_files: bool,
1357    // Amplified user IO request batch size to read data from remote storage backend / local cache.
1358    // ZERO value means, amplifying user io is not enabled.
1359    #[serde(rename = "amplify_io", default = "default_user_io_batch_size")]
1360    pub user_io_batch_size: usize,
1361}
1362
1363impl TryFrom<RafsConfig> for ConfigV2 {
1364    type Error = std::io::Error;
1365
1366    fn try_from(v: RafsConfig) -> std::result::Result<Self, Self::Error> {
1367        let mut backend: BackendConfigV2 = (&v.device.backend).try_into()?;
1368        let mut cache: CacheConfigV2 = (&v.device.cache).try_into()?;
1369        let rafs = RafsConfigV2 {
1370            mode: v.mode,
1371            user_io_batch_size: v.user_io_batch_size,
1372            validate: v.digest_validate,
1373            enable_xattr: v.enable_xattr,
1374            iostats_files: v.iostats_files,
1375            access_pattern: v.access_pattern,
1376            latest_read_files: v.latest_read_files,
1377            prefetch: v.fs_prefetch.into(),
1378        };
1379        if !cache.prefetch.enable && rafs.prefetch.enable {
1380            cache.prefetch = rafs.prefetch.clone();
1381        }
1382
1383        // If prefetch is enabled, disable token refresh by default
1384        if cache.prefetch.enable {
1385            if let Some(registry) = backend.registry.as_mut() {
1386                registry.disable_token_refresh = true;
1387            }
1388        }
1389
1390        Ok(ConfigV2 {
1391            version: 2,
1392            id: v.device.id,
1393            backend: Some(backend),
1394            external_backends: v.device.external_backends,
1395            cache: Some(cache),
1396            rafs: Some(rafs),
1397            overlay: None,
1398            internal: ConfigV2Internal::default(),
1399        })
1400    }
1401}
1402
1403/// Configuration information for filesystem data prefetch.
1404#[derive(Clone, Default, Deserialize)]
1405struct FsPrefetchControl {
1406    /// Whether the filesystem layer data prefetch is enabled or not.
1407    #[serde(default)]
1408    pub enable: bool,
1409
1410    /// How many working threads to prefetch data.
1411    #[serde(default = "default_prefetch_threads_count")]
1412    pub threads_count: usize,
1413
1414    /// The amplify batch size to prefetch data from backend.
1415    #[serde(rename = "merging_size", default = "default_prefetch_batch_size")]
1416    pub batch_size: usize,
1417
1418    /// Network bandwidth limitation for prefetching.
1419    ///
1420    /// In unit of Bytes. It sets a limit to prefetch bandwidth usage in order to
1421    /// reduce congestion with normal user IO.
1422    /// bandwidth_limit == 0 -- prefetch bandwidth ratelimit disabled
1423    /// bandwidth_limit > 0  -- prefetch bandwidth ratelimit enabled.
1424    ///                        Please note that if the value is less than Rafs chunk size,
1425    ///                        it will be raised to the chunk size.
1426    #[serde(default, rename = "bandwidth_rate")]
1427    pub bandwidth_limit: u32,
1428
1429    /// Whether to prefetch all filesystem data.
1430    #[serde(default = "default_prefetch_all")]
1431    pub prefetch_all: bool,
1432}
1433
1434impl From<FsPrefetchControl> for PrefetchConfigV2 {
1435    fn from(v: FsPrefetchControl) -> Self {
1436        PrefetchConfigV2 {
1437            enable: v.enable,
1438            threads_count: v.threads_count,
1439            batch_size: v.batch_size,
1440            bandwidth_limit: v.bandwidth_limit,
1441            prefetch_all: v.prefetch_all,
1442        }
1443    }
1444}
1445
1446/// Configuration information for blob data prefetching.
1447#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
1448struct BlobPrefetchConfig {
1449    /// Whether to enable blob data prefetching.
1450    pub enable: bool,
1451    /// Number of data prefetching working threads.
1452    pub threads_count: usize,
1453    /// The amplify batch size to prefetch data from backend.
1454    #[serde(rename = "merging_size")]
1455    pub batch_size: usize,
1456    /// Network bandwidth rate limit in unit of Bytes and Zero means no limit.
1457    #[serde(rename = "bandwidth_rate")]
1458    pub bandwidth_limit: u32,
1459}
1460
1461impl From<&BlobPrefetchConfig> for PrefetchConfigV2 {
1462    fn from(v: &BlobPrefetchConfig) -> Self {
1463        PrefetchConfigV2 {
1464            enable: v.enable,
1465            threads_count: v.threads_count,
1466            batch_size: v.batch_size,
1467            bandwidth_limit: v.bandwidth_limit,
1468            prefetch_all: true,
1469        }
1470    }
1471}
1472
1473/// Configuration information for a cached blob, corresponding to `FactoryConfig`.
1474#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1475pub(crate) struct BlobCacheEntryConfig {
1476    /// Identifier for the blob cache configuration: corresponding to `FactoryConfig::id`.
1477    #[serde(default)]
1478    id: String,
1479    /// Type of storage backend, corresponding to `FactoryConfig::BackendConfig::backend_type`.
1480    backend_type: String,
1481    /// Configuration for storage backend, corresponding to `FactoryConfig::BackendConfig::backend_config`.
1482    ///
1483    /// Possible value: `LocalFsConfig`, `RegistryConfig`, `OssConfig`, `LocalDiskConfig`.
1484    backend_config: Value,
1485    /// Configuration for external storage backends, order insensitivity.
1486    #[serde(default)]
1487    external_backends: Vec<ExternalBackendConfig>,
1488    /// Type of blob cache, corresponding to `FactoryConfig::CacheConfig::cache_type`.
1489    ///
1490    /// Possible value: "fscache", "filecache".
1491    cache_type: String,
1492    /// Configuration for blob cache, corresponding to `FactoryConfig::CacheConfig::cache_config`.
1493    ///
1494    /// Possible value: `FileCacheConfig`, `FsCacheConfig`.
1495    cache_config: Value,
1496    /// Configuration for data prefetch.
1497    #[serde(default)]
1498    prefetch_config: BlobPrefetchConfig,
1499    /// Optional file path for metadata blobs.
1500    #[serde(default)]
1501    metadata_path: Option<String>,
1502}
1503
1504impl TryFrom<&BlobCacheEntryConfig> for BlobCacheEntryConfigV2 {
1505    type Error = std::io::Error;
1506
1507    fn try_from(v: &BlobCacheEntryConfig) -> std::result::Result<Self, Self::Error> {
1508        let backend_config = BackendConfig {
1509            backend_type: v.backend_type.clone(),
1510            backend_config: v.backend_config.clone(),
1511        };
1512        let cache_config = CacheConfig {
1513            cache_type: v.cache_type.clone(),
1514            cache_compressed: false,
1515            cache_config: v.cache_config.clone(),
1516            cache_validate: false,
1517            prefetch_config: v.prefetch_config.clone(),
1518        };
1519        let mut backend: BackendConfigV2 = (&backend_config).try_into()?;
1520
1521        // If prefetch is enabled, disable token refresh by default
1522        if cache_config.prefetch_config.enable {
1523            if let Some(registry) = backend.registry.as_mut() {
1524                registry.disable_token_refresh = true;
1525            }
1526        }
1527
1528        Ok(BlobCacheEntryConfigV2 {
1529            version: 2,
1530            id: v.id.clone(),
1531            backend,
1532            external_backends: v.external_backends.clone(),
1533            cache: (&cache_config).try_into()?,
1534            metadata_path: v.metadata_path.clone(),
1535        })
1536    }
1537}
1538
1539/// Configuration information for Overlay filesystem.
1540/// OverlayConfig is used to configure the writable layer(upper layer),
1541/// The filesystem will be writable when OverlayConfig is set.
1542#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
1543pub struct OverlayConfig {
1544    pub upper_dir: String,
1545    pub work_dir: String,
1546}
1547
1548#[cfg(test)]
1549mod tests {
1550    use super::*;
1551    use crate::{BlobCacheEntry, BLOB_CACHE_TYPE_META_BLOB};
1552
1553    #[test]
1554    fn test_blob_prefetch_config() {
1555        let config = BlobPrefetchConfig::default();
1556        assert!(!config.enable);
1557        assert_eq!(config.threads_count, 0);
1558        assert_eq!(config.batch_size, 0);
1559        assert_eq!(config.bandwidth_limit, 0);
1560
1561        let content = r#"{
1562            "enable": true,
1563            "threads_count": 2,
1564            "merging_size": 4,
1565            "bandwidth_rate": 5
1566        }"#;
1567        let config: BlobPrefetchConfig = serde_json::from_str(content).unwrap();
1568        assert!(config.enable);
1569        assert_eq!(config.threads_count, 2);
1570        assert_eq!(config.batch_size, 4);
1571        assert_eq!(config.bandwidth_limit, 5);
1572
1573        let config: PrefetchConfigV2 = (&config).into();
1574        assert!(config.enable);
1575        assert_eq!(config.threads_count, 2);
1576        assert_eq!(config.batch_size, 4);
1577        assert_eq!(config.bandwidth_limit, 5);
1578        assert!(config.prefetch_all);
1579    }
1580
1581    #[test]
1582    fn test_file_cache_config() {
1583        let config: FileCacheConfig = serde_json::from_str("{}").unwrap();
1584        assert_eq!(&config.work_dir, ".");
1585        assert!(!config.disable_indexed_map);
1586
1587        let config: FileCacheConfig =
1588            serde_json::from_str("{\"work_dir\":\"/tmp\",\"disable_indexed_map\":true}").unwrap();
1589        assert_eq!(&config.work_dir, "/tmp");
1590        assert!(config.get_work_dir().is_ok());
1591        assert!(config.disable_indexed_map);
1592
1593        let config: FileCacheConfig =
1594            serde_json::from_str("{\"work_dir\":\"/proc/mounts\",\"disable_indexed_map\":true}")
1595                .unwrap();
1596        assert!(config.get_work_dir().is_err());
1597    }
1598
1599    #[test]
1600    fn test_fs_cache_config() {
1601        let config: FsCacheConfig = serde_json::from_str("{}").unwrap();
1602        assert_eq!(&config.work_dir, ".");
1603
1604        let config: FileCacheConfig = serde_json::from_str("{\"work_dir\":\"/tmp\"}").unwrap();
1605        assert_eq!(&config.work_dir, "/tmp");
1606        assert!(config.get_work_dir().is_ok());
1607
1608        let config: FileCacheConfig =
1609            serde_json::from_str("{\"work_dir\":\"/proc/mounts\"}").unwrap();
1610        assert!(config.get_work_dir().is_err());
1611    }
1612
1613    #[test]
1614    fn test_blob_cache_entry() {
1615        let content = r#"{
1616            "type": "bootstrap",
1617            "id": "blob1",
1618            "config": {
1619                "id": "cache1",
1620                "backend_type": "localfs",
1621                "backend_config": {},
1622                "cache_type": "fscache",
1623                "cache_config": {},
1624                "prefetch_config": {
1625                    "enable": true,
1626                    "threads_count": 2,
1627                    "merging_size": 4,
1628                    "bandwidth_rate": 5
1629                },
1630                "metadata_path": "/tmp/metadata1"
1631            },
1632            "domain_id": "domain1"
1633        }"#;
1634        let config: BlobCacheEntry = serde_json::from_str(content).unwrap();
1635        assert_eq!(&config.blob_type, BLOB_CACHE_TYPE_META_BLOB);
1636        assert_eq!(&config.blob_id, "blob1");
1637        assert_eq!(&config.domain_id, "domain1");
1638
1639        let blob_config = config.blob_config_legacy.as_ref().unwrap();
1640        assert_eq!(blob_config.id, "cache1");
1641        assert_eq!(blob_config.backend_type, "localfs");
1642        assert_eq!(blob_config.cache_type, "fscache");
1643        assert!(blob_config.cache_config.is_object());
1644        assert!(blob_config.prefetch_config.enable);
1645        assert_eq!(blob_config.prefetch_config.threads_count, 2);
1646        assert_eq!(blob_config.prefetch_config.batch_size, 4);
1647        assert_eq!(
1648            blob_config.metadata_path.as_ref().unwrap().as_str(),
1649            "/tmp/metadata1"
1650        );
1651
1652        let blob_config: BlobCacheEntryConfigV2 = blob_config.try_into().unwrap();
1653        assert_eq!(blob_config.id, "cache1");
1654        assert_eq!(blob_config.backend.backend_type, "localfs");
1655        assert_eq!(blob_config.cache.cache_type, "fscache");
1656        assert!(blob_config.cache.fs_cache.is_some());
1657        assert!(blob_config.cache.prefetch.enable);
1658        assert_eq!(blob_config.cache.prefetch.threads_count, 2);
1659        assert_eq!(blob_config.cache.prefetch.batch_size, 4);
1660        assert_eq!(
1661            blob_config.metadata_path.as_ref().unwrap().as_str(),
1662            "/tmp/metadata1"
1663        );
1664
1665        let content = r#"{
1666            "type": "bootstrap",
1667            "id": "blob1",
1668            "config": {
1669                "id": "cache1",
1670                "backend_type": "localfs",
1671                "backend_config": {},
1672                "cache_type": "fscache",
1673                "cache_config": {},
1674                "metadata_path": "/tmp/metadata1"
1675            },
1676            "domain_id": "domain1"
1677        }"#;
1678        let config: BlobCacheEntry = serde_json::from_str(content).unwrap();
1679        let blob_config = config.blob_config_legacy.as_ref().unwrap();
1680        assert!(!blob_config.prefetch_config.enable);
1681        assert_eq!(blob_config.prefetch_config.threads_count, 0);
1682        assert_eq!(blob_config.prefetch_config.batch_size, 0);
1683    }
1684
1685    #[test]
1686    fn test_proxy_config() {
1687        let content = r#"{
1688            "url": "foo.com",
1689            "ping_url": "ping.foo.com",
1690            "fallback": true
1691        }"#;
1692        let config: ProxyConfig = serde_json::from_str(content).unwrap();
1693        assert_eq!(config.url, "foo.com");
1694        assert_eq!(config.ping_url, "ping.foo.com");
1695        assert!(config.fallback);
1696        assert_eq!(config.check_interval, 5);
1697    }
1698
1699    #[test]
1700    fn test_oss_config() {
1701        let content = r#"{
1702            "endpoint": "test",
1703            "access_key_id": "test",
1704            "access_key_secret": "test",
1705            "bucket_name": "antsys-nydus",
1706            "object_prefix":"nydus_v2/"
1707        }"#;
1708        let config: OssConfig = serde_json::from_str(content).unwrap();
1709        assert_eq!(config.scheme, "https");
1710        assert!(!config.skip_verify);
1711        assert_eq!(config.timeout, 5);
1712        assert_eq!(config.connect_timeout, 5);
1713    }
1714
1715    #[test]
1716    fn test_s3_config() {
1717        let content = r#"{
1718            "endpoint": "test",
1719            "region": "us-east-1",
1720            "access_key_id": "test",
1721            "access_key_secret": "test",
1722            "bucket_name": "antsys-nydus",
1723            "object_prefix":"nydus_v2/"
1724        }"#;
1725        let config: OssConfig = serde_json::from_str(content).unwrap();
1726        assert_eq!(config.scheme, "https");
1727        assert!(!config.skip_verify);
1728        assert_eq!(config.timeout, 5);
1729        assert_eq!(config.connect_timeout, 5);
1730    }
1731
1732    #[test]
1733    fn test_registry_config() {
1734        let content = r#"{
1735	    "scheme": "http",
1736            "skip_verify": true,
1737	    "host": "my-registry:5000",
1738	    "repo": "test/repo",
1739	    "auth": "base64_encoded_auth",
1740	    "registry_token": "bearer_token",
1741	    "blob_redirected_host": "blob_redirected_host"
1742        }"#;
1743        let config: RegistryConfig = serde_json::from_str(content).unwrap();
1744        assert_eq!(config.scheme, "http");
1745        assert!(config.skip_verify);
1746    }
1747
1748    #[test]
1749    fn test_localfs_config() {
1750        let content = r#"{
1751            "blob_file": "blob_file",
1752            "dir": "blob_dir",
1753            "alt_dirs": ["dir1", "dir2"]
1754        }"#;
1755        let config: LocalFsConfig = serde_json::from_str(content).unwrap();
1756        assert_eq!(config.blob_file, "blob_file");
1757        assert_eq!(config.dir, "blob_dir");
1758        assert_eq!(config.alt_dirs, vec!["dir1", "dir2"]);
1759    }
1760
1761    #[test]
1762    fn test_localdisk_config() {
1763        let content = r#"{
1764            "device_path": "device_path"
1765        }"#;
1766        let config: LocalDiskConfig = serde_json::from_str(content).unwrap();
1767        assert_eq!(config.device_path, "device_path");
1768    }
1769
1770    #[test]
1771    fn test_backend_config() {
1772        let config = BackendConfig {
1773            backend_type: "localfs".to_string(),
1774            backend_config: Default::default(),
1775        };
1776        let str_val = serde_json::to_string(&config).unwrap();
1777        let config2 = serde_json::from_str(&str_val).unwrap();
1778
1779        assert_eq!(config, config2);
1780    }
1781
1782    #[test]
1783    fn test_v2_version() {
1784        let content = "version=2";
1785        let config: ConfigV2 = toml::from_str(content).unwrap();
1786        assert_eq!(config.version, 2);
1787        assert!(config.backend.is_none());
1788    }
1789
1790    #[test]
1791    fn test_v2_backend() {
1792        let content = r#"version=2
1793        [backend]
1794        type = "localfs"
1795        "#;
1796        let config: ConfigV2 = toml::from_str(content).unwrap();
1797        assert_eq!(config.version, 2);
1798        assert!(config.backend.is_some());
1799        assert!(config.cache.is_none());
1800
1801        let backend = config.backend.as_ref().unwrap();
1802        assert_eq!(&backend.backend_type, "localfs");
1803        assert!(backend.localfs.is_none());
1804        assert!(backend.oss.is_none());
1805        assert!(backend.registry.is_none());
1806    }
1807
1808    #[test]
1809    fn test_v2_backend_localfs() {
1810        let content = r#"version=2
1811        [backend]
1812        type = "localfs"
1813        [backend.localfs]
1814        blob_file = "/tmp/nydus.blob.data"
1815        dir = "/tmp"
1816        alt_dirs = ["/var/nydus/cache"]
1817        "#;
1818        let config: ConfigV2 = toml::from_str(content).unwrap();
1819        assert_eq!(config.version, 2);
1820        assert!(config.backend.is_some());
1821
1822        let backend = config.backend.as_ref().unwrap();
1823        assert_eq!(&backend.backend_type, "localfs");
1824        assert!(backend.localfs.is_some());
1825
1826        let localfs = backend.localfs.as_ref().unwrap();
1827        assert_eq!(&localfs.blob_file, "/tmp/nydus.blob.data");
1828        assert_eq!(&localfs.dir, "/tmp");
1829        assert_eq!(&localfs.alt_dirs[0], "/var/nydus/cache");
1830    }
1831
1832    #[test]
1833    fn test_v2_backend_oss() {
1834        let content = r#"version=2
1835        [backend]
1836        type = "oss"
1837        [backend.oss]
1838        endpoint = "my_endpoint"
1839        bucket_name = "my_bucket_name"
1840        object_prefix = "my_object_prefix"
1841        access_key_id = "my_access_key_id"
1842        access_key_secret = "my_access_key_secret"
1843        scheme = "http"
1844        skip_verify = true
1845        timeout = 10
1846        connect_timeout = 10
1847        retry_limit = 5
1848        [backend.oss.proxy]
1849        url = "localhost:6789"
1850        ping_url = "localhost:6789/ping"
1851        fallback = true
1852        check_interval = 10
1853        use_http = true
1854        "#;
1855        let config: ConfigV2 = toml::from_str(content).unwrap();
1856        assert_eq!(config.version, 2);
1857        assert!(config.backend.is_some());
1858        assert!(config.rafs.is_none());
1859
1860        let backend = config.backend.as_ref().unwrap();
1861        assert_eq!(&backend.backend_type, "oss");
1862        assert!(backend.oss.is_some());
1863
1864        let oss = backend.oss.as_ref().unwrap();
1865        assert_eq!(&oss.endpoint, "my_endpoint");
1866        assert_eq!(&oss.bucket_name, "my_bucket_name");
1867        assert_eq!(&oss.object_prefix, "my_object_prefix");
1868        assert_eq!(&oss.access_key_id, "my_access_key_id");
1869        assert_eq!(&oss.access_key_secret, "my_access_key_secret");
1870        assert_eq!(&oss.scheme, "http");
1871        assert!(oss.skip_verify);
1872        assert_eq!(oss.timeout, 10);
1873        assert_eq!(oss.connect_timeout, 10);
1874        assert_eq!(oss.retry_limit, 5);
1875        assert_eq!(&oss.proxy.url, "localhost:6789");
1876        assert_eq!(&oss.proxy.ping_url, "localhost:6789/ping");
1877        assert_eq!(oss.proxy.check_interval, 10);
1878        assert!(oss.proxy.fallback);
1879        assert!(oss.proxy.use_http);
1880    }
1881
1882    #[test]
1883    fn test_v2_backend_registry() {
1884        let content = r#"version=2
1885        [backend]
1886        type = "registry"
1887        [backend.registry]
1888        scheme = "http"
1889        host = "localhost"
1890        repo = "nydus"
1891        auth = "auth"
1892        skip_verify = true
1893        timeout = 10
1894        connect_timeout = 10
1895        retry_limit = 5
1896        registry_token = "bear_token"
1897        blob_url_scheme = "https"
1898        blob_redirected_host = "redirect.registry.com"
1899        [backend.registry.proxy]
1900        url = "localhost:6789"
1901        ping_url = "localhost:6789/ping"
1902        fallback = true
1903        check_interval = 10
1904        use_http = true
1905        "#;
1906        let config: ConfigV2 = toml::from_str(content).unwrap();
1907        assert_eq!(config.version, 2);
1908        assert!(config.backend.is_some());
1909        assert!(config.rafs.is_none());
1910
1911        let backend = config.backend.as_ref().unwrap();
1912        assert_eq!(&backend.backend_type, "registry");
1913        assert!(backend.registry.is_some());
1914
1915        let registry = backend.registry.as_ref().unwrap();
1916        assert_eq!(&registry.scheme, "http");
1917        assert_eq!(&registry.host, "localhost");
1918        assert_eq!(&registry.repo, "nydus");
1919        assert_eq!(registry.auth.as_ref().unwrap(), "auth");
1920        assert!(registry.skip_verify);
1921        assert_eq!(registry.timeout, 10);
1922        assert_eq!(registry.connect_timeout, 10);
1923        assert_eq!(registry.retry_limit, 5);
1924        assert_eq!(registry.registry_token.as_ref().unwrap(), "bear_token");
1925        assert_eq!(registry.blob_url_scheme, "https");
1926        assert_eq!(registry.blob_redirected_host, "redirect.registry.com");
1927
1928        assert_eq!(&registry.proxy.url, "localhost:6789");
1929        assert_eq!(&registry.proxy.ping_url, "localhost:6789/ping");
1930        assert_eq!(registry.proxy.check_interval, 10);
1931        assert!(registry.proxy.fallback);
1932        assert!(registry.proxy.use_http);
1933    }
1934
1935    #[test]
1936    fn test_v2_cache() {
1937        let content = r#"version=2
1938        [cache]
1939        type = "filecache"
1940        compressed = true
1941        validate = true
1942        [cache.filecache]
1943        work_dir = "/tmp"
1944        [cache.fscache]
1945        work_dir = "./"
1946        [cache.prefetch]
1947        enable = true
1948        threads = 8
1949        batch_size = 1000000
1950        bandwidth_limit = 10000000
1951        "#;
1952        let config: ConfigV2 = toml::from_str(content).unwrap();
1953        assert_eq!(config.version, 2);
1954        assert!(config.backend.is_none());
1955        assert!(config.rafs.is_none());
1956        assert!(config.cache.is_some());
1957
1958        let cache = config.cache.as_ref().unwrap();
1959        assert_eq!(&cache.cache_type, "filecache");
1960        assert!(cache.cache_compressed);
1961        assert!(cache.cache_validate);
1962        let filecache = cache.file_cache.as_ref().unwrap();
1963        assert_eq!(&filecache.work_dir, "/tmp");
1964        let fscache = cache.fs_cache.as_ref().unwrap();
1965        assert_eq!(&fscache.work_dir, "./");
1966
1967        let prefetch = &cache.prefetch;
1968        assert!(prefetch.enable);
1969        assert_eq!(prefetch.threads_count, 8);
1970        assert_eq!(prefetch.batch_size, 1000000);
1971        assert_eq!(prefetch.bandwidth_limit, 10000000);
1972    }
1973
1974    #[test]
1975    fn test_v2_rafs() {
1976        let content = r#"version=2
1977        [rafs]
1978        mode = "direct"
1979        batch_size = 1000000
1980        validate = true
1981        enable_xattr = true
1982        iostats_files = true
1983        access_pattern = true
1984        latest_read_files = true
1985        [rafs.prefetch]
1986        enable = true
1987        threads = 4
1988        batch_size = 1000000
1989        bandwidth_limit = 10000000
1990        prefetch_all = true
1991        "#;
1992        let config: ConfigV2 = toml::from_str(content).unwrap();
1993        assert_eq!(config.version, 2);
1994        assert!(config.backend.is_none());
1995        assert!(config.cache.is_none());
1996        assert!(config.rafs.is_some());
1997
1998        let rafs = config.rafs.as_ref().unwrap();
1999        assert_eq!(&rafs.mode, "direct");
2000        assert_eq!(rafs.user_io_batch_size, 1000000);
2001        assert!(rafs.validate);
2002        assert!(rafs.enable_xattr);
2003        assert!(rafs.iostats_files);
2004        assert!(rafs.access_pattern);
2005        assert!(rafs.latest_read_files);
2006        assert!(rafs.prefetch.enable);
2007        assert_eq!(rafs.prefetch.threads_count, 4);
2008        assert_eq!(rafs.prefetch.batch_size, 1000000);
2009        assert_eq!(rafs.prefetch.bandwidth_limit, 10000000);
2010        assert!(rafs.prefetch.prefetch_all)
2011    }
2012
2013    #[test]
2014    fn test_v2_blob_cache_entry() {
2015        let content = r#"version=2
2016        id = "my_id"
2017        metadata_path = "meta_path"
2018        [backend]
2019        type = "localfs"
2020        [backend.localfs]
2021        blob_file = "/tmp/nydus.blob.data"
2022        dir = "/tmp"
2023        alt_dirs = ["/var/nydus/cache"]
2024        [cache]
2025        type = "filecache"
2026        compressed = true
2027        validate = true
2028        [cache.filecache]
2029        work_dir = "/tmp"
2030        "#;
2031        let config: BlobCacheEntryConfigV2 = toml::from_str(content).unwrap();
2032        assert_eq!(config.version, 2);
2033        assert_eq!(&config.id, "my_id");
2034        assert_eq!(config.metadata_path.as_ref().unwrap(), "meta_path");
2035
2036        let backend = &config.backend;
2037        assert_eq!(&backend.backend_type, "localfs");
2038        assert!(backend.localfs.is_some());
2039
2040        let localfs = backend.localfs.as_ref().unwrap();
2041        assert_eq!(&localfs.blob_file, "/tmp/nydus.blob.data");
2042        assert_eq!(&localfs.dir, "/tmp");
2043        assert_eq!(&localfs.alt_dirs[0], "/var/nydus/cache");
2044    }
2045
2046    #[test]
2047    fn test_sample_config_file() {
2048        let content = r#"{
2049            "device": {
2050                "backend": {
2051                    "type": "localfs",
2052                    "config": {
2053                        "dir": "/tmp/AM7TxD/blobs",
2054                        "readahead": true
2055                    }
2056                },
2057                "cache": {
2058                    "type": "blobcache",
2059                    "compressed": true,
2060                    "config": {
2061                        "work_dir": "/tmp/AM7TxD/cache"
2062                    }
2063                }
2064            },
2065            "mode": "cached",
2066            "digest_validate": true,
2067            "iostats_files": false
2068        }
2069        "#;
2070        let config = ConfigV2::from_str(content).unwrap();
2071        assert_eq!(&config.id, "");
2072    }
2073
2074    #[test]
2075    fn test_snapshotter_sample_config() {
2076        let content = r#"
2077        {
2078            "device": {
2079                "backend": {
2080                    "type": "registry",
2081                    "config": {
2082                        "readahead": false,
2083                        "host": "localhost",
2084                        "repo": "vke/golang",
2085                        "auth": "",
2086                        "scheme": "https",
2087                        "proxy": {
2088                            "fallback": false
2089                        },
2090                        "timeout": 5,
2091                        "connect_timeout": 5,
2092                        "retry_limit": 2
2093                    }
2094                },
2095                "cache": {
2096                    "type": "blobcache",
2097                    "compressed": true,
2098                    "config": {
2099                        "work_dir": "/var/lib/containerd/io.containerd.snapshotter.v1.nydus/cache",
2100                        "disable_indexed_map": false
2101                    }
2102                }
2103            },
2104            "mode": "direct",
2105            "digest_validate": false,
2106            "enable_xattr": true,
2107            "fs_prefetch": {
2108                "enable": true,
2109                "prefetch_all": true,
2110                "threads_count": 8,
2111                "merging_size": 1048576,
2112                "bandwidth_rate": 0
2113            }
2114        }
2115        "#;
2116        let config = ConfigV2::from_str(content).unwrap();
2117        assert_eq!(&config.id, "");
2118        // token refresh should be disabled when prefetch is enabled
2119        assert!(
2120            config
2121                .get_backend_config()
2122                .unwrap()
2123                .get_registry_config()
2124                .unwrap()
2125                .disable_token_refresh
2126        );
2127    }
2128
2129    #[test]
2130    fn test_backend_http_proxy_config() {
2131        let config =
2132            r#"{"version":2,"backend":{"type":"http-proxy","http-proxy":{"addr":"/tmp"}}}"#;
2133        let config = ConfigV2::from_str(config).unwrap();
2134        let backend = config.backend.unwrap();
2135        assert_eq!(&backend.backend_type, "http-proxy");
2136        assert_eq!(&backend.http_proxy.unwrap().addr, "/tmp");
2137    }
2138
2139    #[test]
2140    fn test_new_localfs() {
2141        let config = ConfigV2::new_localfs("id1", "./").unwrap();
2142        assert_eq!(&config.id, "id1");
2143        assert_eq!(config.backend.as_ref().unwrap().backend_type, "localfs");
2144    }
2145
2146    #[test]
2147    fn test_update_registry_auth_info() {
2148        let config = r#"
2149        {
2150            "device": {
2151              "id": "test",
2152              "backend": {
2153                "type": "registry",
2154                "config": {
2155                    "readahead": false,
2156                    "host": "docker.io",
2157                    "repo": "library/nginx",
2158                    "scheme": "https",
2159                    "proxy": {
2160                        "fallback": false
2161                    },
2162                    "timeout": 5,
2163                    "connect_timeout": 5,
2164                    "retry_limit": 8
2165                }
2166              }
2167            },
2168            "mode": "direct",
2169            "digest_validate": false,
2170            "enable_xattr": true,
2171            "fs_prefetch": {
2172              "enable": true,
2173              "threads_count": 10,
2174              "merging_size": 131072,
2175              "bandwidth_rate": 10485760
2176            }
2177          }"#;
2178
2179        let mut rafs_config = ConfigV2::from_str(&config).unwrap();
2180        let test_auth = "test_auth".to_string();
2181
2182        rafs_config.update_registry_auth_info(&Some(test_auth.clone()));
2183
2184        let backend = rafs_config.backend.unwrap();
2185        let registry = backend.registry.unwrap();
2186        let auth = registry.auth.unwrap();
2187        assert_eq!(auth, test_auth);
2188    }
2189
2190    #[test]
2191    fn test_config2_error() {
2192        let content_bad_version = r#"version=3
2193        "#;
2194        let cfg: ConfigV2 = toml::from_str(content_bad_version).unwrap();
2195        assert!(!cfg.validate());
2196        let cfg = ConfigV2::new("id");
2197        assert!(cfg.get_backend_config().is_err());
2198        assert!(cfg.get_cache_config().is_err());
2199        assert!(cfg.get_rafs_config().is_err());
2200        assert!(cfg.get_cache_working_directory().is_err());
2201
2202        let content = r#"version=2
2203            [cache]
2204            type = "filecache"
2205            [cache.filecache]
2206            work_dir = "/tmp"
2207        "#;
2208        let cfg: ConfigV2 = toml::from_str(content).unwrap();
2209        assert_eq!(cfg.get_cache_working_directory().unwrap(), "/tmp");
2210
2211        let content = r#"version=2
2212            [cache]
2213            type = "fscache"
2214            [cache.fscache]
2215            work_dir = "./foo"
2216        "#;
2217        let cfg: ConfigV2 = toml::from_str(content).unwrap();
2218        assert_eq!(cfg.get_cache_working_directory().unwrap(), "./foo");
2219
2220        let content = r#"version=2
2221            [cache]
2222            type = "bar"
2223        "#;
2224        let cfg: ConfigV2 = toml::from_str(content).unwrap();
2225        assert!(cfg.get_cache_working_directory().is_err());
2226
2227        let content = r#"
2228            foo-bar-xxxx
2229        "#;
2230        assert!(toml::from_str::<ConfigV2>(content).is_err());
2231    }
2232
2233    #[test]
2234    fn test_backend_config_valid() {
2235        let mut cfg = BackendConfigV2 {
2236            backend_type: "localdisk".to_string(),
2237            ..Default::default()
2238        };
2239        assert!(!cfg.validate());
2240        cfg.localdisk = Some(LocalDiskConfig {
2241            device_path: "".to_string(),
2242            disable_gpt: true,
2243        });
2244        assert!(!cfg.validate());
2245
2246        let cfg = BackendConfigV2 {
2247            backend_type: "localfs".to_string(),
2248            ..Default::default()
2249        };
2250        assert!(!cfg.validate());
2251
2252        let cfg = BackendConfigV2 {
2253            backend_type: "oss".to_string(),
2254            ..Default::default()
2255        };
2256        assert!(!cfg.validate());
2257
2258        let cfg = BackendConfigV2 {
2259            backend_type: "s3".to_string(),
2260            ..Default::default()
2261        };
2262        assert!(!cfg.validate());
2263
2264        let cfg = BackendConfigV2 {
2265            backend_type: "register".to_string(),
2266            ..Default::default()
2267        };
2268        assert!(!cfg.validate());
2269
2270        let cfg = BackendConfigV2 {
2271            backend_type: "http-proxy".to_string(),
2272            ..Default::default()
2273        };
2274        assert!(!cfg.validate());
2275
2276        let cfg = BackendConfigV2 {
2277            backend_type: "foobar".to_string(),
2278            ..Default::default()
2279        };
2280        assert!(!cfg.validate());
2281    }
2282
2283    fn get_config(backend_type: &str) {
2284        let mut cfg: BackendConfigV2 = BackendConfigV2::default();
2285        assert!(cfg.get_localdisk_config().is_err());
2286
2287        cfg.backend_type = backend_type.to_string();
2288        assert!(cfg.get_localdisk_config().is_err());
2289    }
2290
2291    #[test]
2292    fn test_get_config() {
2293        get_config("localdisk");
2294        get_config("localfs");
2295        get_config("oss");
2296        get_config("s3");
2297        get_config("register");
2298        get_config("http-proxy");
2299    }
2300
2301    #[test]
2302    fn test_cache_config_valid() {
2303        let cfg = CacheConfigV2 {
2304            cache_type: "blobcache".to_string(),
2305            ..Default::default()
2306        };
2307        assert!(!cfg.validate());
2308
2309        let cfg = CacheConfigV2 {
2310            cache_type: "fscache".to_string(),
2311            ..Default::default()
2312        };
2313        assert!(!cfg.validate());
2314
2315        let cfg = CacheConfigV2 {
2316            cache_type: "dummycache".to_string(),
2317            ..Default::default()
2318        };
2319        assert!(cfg.validate());
2320
2321        let cfg = CacheConfigV2 {
2322            cache_type: "foobar".to_string(),
2323            ..Default::default()
2324        };
2325        assert!(!cfg.validate());
2326    }
2327
2328    #[test]
2329    fn test_get_fscache_config() {
2330        let mut cfg = CacheConfigV2::default();
2331        assert!(cfg.get_fscache_config().is_err());
2332        cfg.cache_type = "fscache".to_string();
2333        assert!(cfg.get_fscache_config().is_err());
2334    }
2335
2336    #[test]
2337    fn test_fscache_get_work_dir() {
2338        let mut cfg = FsCacheConfig::default();
2339        assert!(cfg.get_work_dir().is_err());
2340        cfg.work_dir = ".".to_string();
2341        assert!(cfg.get_work_dir().is_ok());
2342        cfg.work_dir = "foobar".to_string();
2343        let res = cfg.get_work_dir().is_ok();
2344        fs::remove_dir_all("foobar").unwrap();
2345        assert!(res);
2346    }
2347
2348    #[test]
2349    fn test_config_v2_from_file() {
2350        let content = r#"version=2
2351            [cache]
2352            type = "filecache"
2353            [cache.filecache]
2354            work_dir = "/tmp"
2355        "#;
2356        if fs::write("test_config_v2_from_file.cfg", content).is_ok() {
2357            let res = ConfigV2::from_file("test_config_v2_from_file.cfg").is_ok();
2358            fs::remove_file("test_config_v2_from_file.cfg").unwrap();
2359            assert!(res);
2360        } else {
2361            assert!(ConfigV2::from_file("test_config_v2_from_file.cfg").is_err());
2362        }
2363    }
2364
2365    #[test]
2366    fn test_blob_cache_entry_v2_from_file() {
2367        let content = r#"version=2
2368        id = "my_id"
2369        metadata_path = "meta_path"
2370        [backend]
2371        type = "localfs"
2372        [backend.localfs]
2373        blob_file = "/tmp/nydus.blob.data"
2374        dir = "/tmp"
2375        alt_dirs = ["/var/nydus/cache"]
2376        [cache]
2377        type = "filecache"
2378        compressed = true
2379        validate = true
2380        [cache.filecache]
2381        work_dir = "/tmp"
2382        "#;
2383        if fs::write("test_blob_cache_entry_v2_from_file.cfg", content).is_ok() {
2384            let res =
2385                BlobCacheEntryConfigV2::from_file("test_blob_cache_entry_v2_from_file.cfg").is_ok();
2386            fs::remove_file("test_blob_cache_entry_v2_from_file.cfg").unwrap();
2387            assert!(res);
2388        } else {
2389            assert!(ConfigV2::from_file("test_blob_cache_entry_v2_from_file.cfg").is_err());
2390        }
2391    }
2392
2393    #[test]
2394    fn test_blob_cache_valid() {
2395        let err_version_content = r#"version=1"#;
2396
2397        let config: BlobCacheEntryConfigV2 = toml::from_str(err_version_content).unwrap();
2398        assert!(!config.validate());
2399
2400        let content = r#"version=2
2401        id = "my_id"
2402        metadata_path = "meta_path"
2403        [backend]
2404        type = "localfs"
2405        [backend.localfs]
2406        blob_file = "/tmp/nydus.blob.data"
2407        dir = "/tmp"
2408        alt_dirs = ["/var/nydus/cache"]
2409        [cache]
2410        type = "filecache"
2411        compressed = true
2412        validate = true
2413        [cache.filecache]
2414        work_dir = "/tmp"
2415        "#;
2416
2417        let config: BlobCacheEntryConfigV2 = toml::from_str(content).unwrap();
2418        assert!(config.validate());
2419    }
2420
2421    #[test]
2422    fn test_blob_from_str() {
2423        let content = r#"version=2
2424        id = "my_id"
2425        metadata_path = "meta_path"
2426        [backend]
2427        type = "localfs"
2428        [backend.localfs]
2429        blob_file = "/tmp/nydus.blob.data"
2430        dir = "/tmp"
2431        alt_dirs = ["/var/nydus/cache"]
2432        [cache]
2433        type = "filecache"
2434        compressed = true
2435        validate = true
2436        [cache.filecache]
2437        work_dir = "/tmp"
2438        "#;
2439
2440        let config: BlobCacheEntryConfigV2 = BlobCacheEntryConfigV2::from_str(content).unwrap();
2441        assert_eq!(config.version, 2);
2442        assert_eq!(config.id, "my_id");
2443        assert_eq!(config.backend.localfs.unwrap().dir, "/tmp");
2444        assert_eq!(config.cache.file_cache.unwrap().work_dir, "/tmp");
2445        let content = r#"
2446            {
2447                "version": 2,
2448                "id": "my_id",
2449                "backend": {
2450                    "type": "localfs",
2451                    "localfs": {
2452                        "dir": "/tmp"
2453                    }
2454                }
2455            }
2456        "#;
2457        let config: BlobCacheEntryConfigV2 = BlobCacheEntryConfigV2::from_str(content).unwrap();
2458
2459        assert_eq!(config.version, 2);
2460        assert_eq!(config.id, "my_id");
2461        assert_eq!(config.backend.localfs.unwrap().dir, "/tmp");
2462
2463        let content = r#"foobar"#;
2464        assert!(BlobCacheEntryConfigV2::from_str(content).is_err());
2465    }
2466
2467    #[test]
2468    fn test_blob_cache_entry_from_file() {
2469        let content = r#"{
2470            "type": "bootstrap",
2471            "id": "blob1",
2472            "config": {
2473                "id": "cache1",
2474                "backend_type": "localfs",
2475                "backend_config": {},
2476                "cache_type": "fscache",
2477                "cache_config": {},
2478                "metadata_path": "/tmp/metadata1"
2479            },
2480            "domain_id": "domain1"
2481        }"#;
2482        if fs::write("test_blob_cache_entry_from_file.cfg", content).is_ok() {
2483            let res = BlobCacheEntry::from_file("test_blob_cache_entry_from_file.cfg").is_ok();
2484            fs::remove_file("test_blob_cache_entry_from_file.cfg").unwrap();
2485            assert!(res);
2486        } else {
2487            assert!(ConfigV2::from_file("test_blob_cache_entry_from_file.cfg").is_err());
2488        }
2489    }
2490
2491    #[test]
2492    fn test_blob_cache_entry_valid() {
2493        let content = r#"{
2494            "type": "bootstrap",
2495            "id": "blob1",
2496            "config": {
2497                "id": "cache1",
2498                "backend_type": "localfs",
2499                "backend_config": {},
2500                "cache_type": "fscache",
2501                "cache_config": {},
2502                "metadata_path": "/tmp/metadata1"
2503            },
2504            "domain_id": "domain1"
2505        }"#;
2506        let mut cfg = BlobCacheEntry::from_str(content).unwrap();
2507        cfg.blob_type = "foobar".to_string();
2508        assert!(!cfg.validate());
2509
2510        let content = r#"{
2511            "type": "bootstrap",
2512            "id": "blob1",
2513            "domain_id": "domain1"
2514        }"#;
2515        let cfg = BlobCacheEntry::from_str(content).unwrap();
2516        assert!(cfg.validate());
2517    }
2518
2519    #[test]
2520    fn test_blob_cache_entry_from_str() {
2521        let content = r#"{
2522            "type": "bootstrap",
2523            "id": "blob1",
2524            "config": {
2525                "id": "cache1",
2526                "backend_type": "localfs",
2527                "backend_config": {},
2528                "cache_type": "fscache",
2529                "cache_config": {},
2530                "metadata_path": "/tmp/metadata1"
2531            },
2532            "domain_id": "domain1"
2533        }"#;
2534        assert!(BlobCacheEntry::from_str(content).is_ok());
2535        let content = r#"{
2536            "type": "foobar",
2537            "id": "blob1",
2538            "config": {
2539                "id": "cache1",
2540                "backend_type": "foobar",
2541                "backend_config": {},
2542                "cache_type": "foobar",
2543                "cache_config": {},
2544                "metadata_path": "/tmp/metadata1"
2545            },
2546            "domain_id": "domain1"
2547        }"#;
2548        assert!(BlobCacheEntry::from_str(content).is_err());
2549
2550        let content = r#"foobar"#;
2551        assert!(BlobCacheEntry::from_str(content).is_err());
2552    }
2553
2554    #[test]
2555    fn test_default_value() {
2556        assert!(default_true());
2557        assert_eq!(default_prefetch_batch_size(), 1024 * 1024);
2558        assert_eq!(default_prefetch_threads_count(), 8);
2559    }
2560
2561    #[test]
2562    fn test_backend_config_try_from() {
2563        let config = BackendConfig {
2564            backend_type: "localdisk".to_string(),
2565            backend_config: serde_json::to_value(LocalDiskConfig::default()).unwrap(),
2566        };
2567        assert!(BackendConfigV2::try_from(&config).is_ok());
2568
2569        let config = BackendConfig {
2570            backend_type: "localfs".to_string(),
2571            backend_config: serde_json::to_value(LocalFsConfig::default()).unwrap(),
2572        };
2573        assert!(BackendConfigV2::try_from(&config).is_ok());
2574
2575        let config = BackendConfig {
2576            backend_type: "oss".to_string(),
2577            backend_config: serde_json::to_value(OssConfig::default()).unwrap(),
2578        };
2579        assert!(BackendConfigV2::try_from(&config).is_ok());
2580
2581        let config = BackendConfig {
2582            backend_type: "s3".to_string(),
2583            backend_config: serde_json::to_value(S3Config::default()).unwrap(),
2584        };
2585        assert!(BackendConfigV2::try_from(&config).is_ok());
2586
2587        let config = BackendConfig {
2588            backend_type: "registry".to_string(),
2589            backend_config: serde_json::to_value(RegistryConfig::default()).unwrap(),
2590        };
2591        assert!(BackendConfigV2::try_from(&config).is_ok());
2592
2593        let config = BackendConfig {
2594            backend_type: "foobar".to_string(),
2595            backend_config: serde_json::to_value(LocalDiskConfig::default()).unwrap(),
2596        };
2597        assert!(BackendConfigV2::try_from(&config).is_err());
2598    }
2599}
nydus_api/config.rs

nydus_api/
config.rs
