rustfs_tls_runtime/
certs.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use rustls::RootCertStore;
16use rustls::server::{
17    ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni, WebPkiClientVerifier, danger::ClientCertVerifier,
18};
19use rustls::sign::CertifiedKey;
20use rustls_pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject};
21use std::collections::HashMap;
22use std::io::Error;
23use std::path::{Path, PathBuf};
24use std::sync::Arc;
25use std::{fs, io};
26use tracing::{debug, warn};
27
28#[derive(Debug, Clone)]
29pub struct CertDirectoryLoadOptions {
30    dir_path: PathBuf,
31    cert_filename: String,
32    key_filename: String,
33}
34
35impl CertDirectoryLoadOptions {
36    pub fn builder(
37        dir_path: impl Into<PathBuf>,
38        cert_filename: impl Into<String>,
39        key_filename: impl Into<String>,
40    ) -> CertDirectoryLoadOptionsBuilder {
41        CertDirectoryLoadOptionsBuilder {
42            dir_path: dir_path.into(),
43            cert_filename: cert_filename.into(),
44            key_filename: key_filename.into(),
45        }
46    }
47
48    fn validate(&self) -> io::Result<()> {
49        if self.cert_filename.is_empty() {
50            return Err(certs_error("certificate filename cannot be empty".to_string()));
51        }
52        if self.key_filename.is_empty() {
53            return Err(certs_error("private key filename cannot be empty".to_string()));
54        }
55        Ok(())
56    }
57}
58
59#[derive(Debug, Clone)]
60pub struct CertDirectoryLoadOptionsBuilder {
61    dir_path: PathBuf,
62    cert_filename: String,
63    key_filename: String,
64}
65
66impl CertDirectoryLoadOptionsBuilder {
67    pub fn cert_filename(mut self, cert_filename: impl Into<String>) -> Self {
68        self.cert_filename = cert_filename.into();
69        self
70    }
71
72    pub fn key_filename(mut self, key_filename: impl Into<String>) -> Self {
73        self.key_filename = key_filename.into();
74        self
75    }
76
77    pub fn build(self) -> CertDirectoryLoadOptions {
78        CertDirectoryLoadOptions {
79            dir_path: self.dir_path,
80            cert_filename: self.cert_filename,
81            key_filename: self.key_filename,
82        }
83    }
84}
85
86#[derive(Debug, Clone)]
87pub struct WebPkiClientVerifierOptions {
88    tls_path: PathBuf,
89    enabled: bool,
90    client_ca_cert_filename: String,
91    fallback_ca_cert_filename: String,
92}
93
94impl WebPkiClientVerifierOptions {
95    pub fn builder(
96        tls_path: impl Into<PathBuf>,
97        client_ca_cert_filename: impl Into<String>,
98        fallback_ca_cert_filename: impl Into<String>,
99    ) -> WebPkiClientVerifierOptionsBuilder {
100        WebPkiClientVerifierOptionsBuilder {
101            tls_path: tls_path.into(),
102            enabled: false,
103            client_ca_cert_filename: client_ca_cert_filename.into(),
104            fallback_ca_cert_filename: fallback_ca_cert_filename.into(),
105        }
106    }
107}
108
109#[derive(Debug, Clone)]
110pub struct WebPkiClientVerifierOptionsBuilder {
111    tls_path: PathBuf,
112    enabled: bool,
113    client_ca_cert_filename: String,
114    fallback_ca_cert_filename: String,
115}
116
117impl WebPkiClientVerifierOptionsBuilder {
118    pub fn enabled(mut self, enabled: bool) -> Self {
119        self.enabled = enabled;
120        self
121    }
122
123    pub fn client_ca_cert_filename(mut self, client_ca_cert_filename: impl Into<String>) -> Self {
124        self.client_ca_cert_filename = client_ca_cert_filename.into();
125        self
126    }
127
128    pub fn fallback_ca_cert_filename(mut self, fallback_ca_cert_filename: impl Into<String>) -> Self {
129        self.fallback_ca_cert_filename = fallback_ca_cert_filename.into();
130        self
131    }
132
133    pub fn build(self) -> WebPkiClientVerifierOptions {
134        WebPkiClientVerifierOptions {
135            tls_path: self.tls_path,
136            enabled: self.enabled,
137            client_ca_cert_filename: self.client_ca_cert_filename,
138            fallback_ca_cert_filename: self.fallback_ca_cert_filename,
139        }
140    }
141}
142
143pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
144    let cert_file = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?;
145    let mut reader = io::BufReader::new(cert_file);
146
147    let certs = CertificateDer::pem_reader_iter(&mut reader)
148        .collect::<Result<Vec<_>, _>>()
149        .map_err(|e| certs_error(format!("certificate file {filename} format error:{e:?}")))?;
150    if certs.is_empty() {
151        return Err(certs_error(format!("No valid certificate was found in the certificate file {filename}")));
152    }
153    Ok(certs)
154}
155
156pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result<Vec<Vec<u8>>> {
157    let pem = fs::read(path)?;
158    let mut reader = io::BufReader::new(&pem[..]);
159
160    let certs = CertificateDer::pem_reader_iter(&mut reader)
161        .collect::<Result<Vec<_>, _>>()
162        .map_err(|e| certs_error(format!("Failed to parse PEM certs from {path}: {e}")))?;
163
164    Ok(certs.into_iter().map(|c| c.to_vec()).collect())
165}
166
167pub fn build_webpki_client_verifier(options: WebPkiClientVerifierOptions) -> io::Result<Option<Arc<dyn ClientCertVerifier>>> {
168    if !options.enabled {
169        return Ok(None);
170    }
171
172    let tls_path = &options.tls_path;
173    let ca_path = mtls_ca_bundle_path(&options).ok_or_else(|| {
174        Error::other(format!(
175            "mTLS is enabled but missing {}/{} (or fallback {}/{})",
176            tls_path.display(),
177            options.client_ca_cert_filename,
178            tls_path.display(),
179            options.fallback_ca_cert_filename
180        ))
181    })?;
182
183    let ca_path = ca_path
184        .to_str()
185        .ok_or_else(|| Error::other(format!("Invalid UTF-8 in mTLS CA path: {ca_path:?}")))?;
186
187    let der_list = load_cert_bundle_der_bytes(ca_path)?;
188
189    let mut store = RootCertStore::empty();
190    for der in der_list {
191        store
192            .add(der.into())
193            .map_err(|e| Error::other(format!("Invalid client CA cert: {e}")))?;
194    }
195
196    let verifier = WebPkiClientVerifier::builder(Arc::new(store))
197        .build()
198        .map_err(|e| Error::other(format!("Build client cert verifier failed: {e}")))?;
199
200    Ok(Some(verifier))
201}
202
203fn mtls_ca_bundle_path(options: &WebPkiClientVerifierOptions) -> Option<PathBuf> {
204    let p1 = options.tls_path.join(&options.client_ca_cert_filename);
205    if p1.exists() {
206        return Some(p1);
207    }
208    let p2 = options.tls_path.join(&options.fallback_ca_cert_filename);
209    if p2.exists() {
210        return Some(p2);
211    }
212    None
213}
214
215pub fn load_private_key(filename: &str) -> io::Result<PrivateKeyDer<'static>> {
216    let keyfile = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?;
217    let mut reader = io::BufReader::new(keyfile);
218
219    PrivateKeyDer::from_pem_reader(&mut reader)
220        .map_err(|e| certs_error(format!("failed to parse private key in {filename}: {e}")))
221}
222
223pub fn certs_error(err: String) -> Error {
224    Error::other(err)
225}
226
227#[derive(Debug, Clone, PartialEq, Eq)]
228pub enum TlsCertPairStatus {
229    MissingBoth,
230    MissingCert,
231    MissingKey,
232    Valid,
233    Invalid { error: String },
234}
235
236impl TlsCertPairStatus {
237    pub fn is_valid(&self) -> bool {
238        matches!(self, Self::Valid)
239    }
240}
241
242#[derive(Debug, Clone, PartialEq, Eq)]
243pub struct TlsCertPairInspection {
244    pub cert_path: PathBuf,
245    pub key_path: PathBuf,
246    pub status: TlsCertPairStatus,
247}
248
249impl TlsCertPairInspection {
250    pub fn is_valid(&self) -> bool {
251        self.status.is_valid()
252    }
253}
254
255#[derive(Debug, Clone, PartialEq, Eq)]
256pub struct TlsDomainInspection {
257    pub domain_name: String,
258    pub pair: TlsCertPairInspection,
259}
260
261#[derive(Debug, Clone, PartialEq, Eq)]
262pub struct TlsDirectoryInspection {
263    pub directory: PathBuf,
264    pub canonical_directory: Option<PathBuf>,
265    pub root_pair: TlsCertPairInspection,
266    pub domain_pairs: Vec<TlsDomainInspection>,
267    pub skipped_directory_names: Vec<String>,
268}
269
270impl TlsDirectoryInspection {
271    pub fn valid_domain_names(&self) -> Vec<&str> {
272        self.domain_pairs
273            .iter()
274            .filter(|entry| entry.pair.is_valid())
275            .map(|entry| entry.domain_name.as_str())
276            .collect()
277    }
278
279    pub fn has_valid_root_pair(&self) -> bool {
280        self.root_pair.is_valid()
281    }
282}
283
284fn is_discoverable_cert_domain_dir(domain_name: &str) -> bool {
285    !domain_name.starts_with('.')
286}
287
288pub fn inspect_cert_directory(options: CertDirectoryLoadOptions) -> io::Result<TlsDirectoryInspection> {
289    options.validate()?;
290
291    let dir = options.dir_path.as_path();
292    if !dir.exists() || !dir.is_dir() {
293        return Err(certs_error(format!(
294            "The certificate directory does not exist or is not a directory: {}",
295            dir.display()
296        )));
297    }
298
299    let root_pair = inspect_cert_key_pair(dir, &options.cert_filename, &options.key_filename);
300    let canonical_directory = fs::canonicalize(dir).ok();
301    let mut domain_pairs = Vec::new();
302    let mut skipped_directory_names = Vec::new();
303
304    for entry in fs::read_dir(dir)? {
305        let entry = entry?;
306        let path = entry.path();
307
308        if !path.is_dir() {
309            continue;
310        }
311
312        let domain_name = path
313            .file_name()
314            .and_then(|name| name.to_str())
315            .ok_or_else(|| certs_error(format!("invalid domain name directory:{path:?}")))?;
316        if !is_discoverable_cert_domain_dir(domain_name) {
317            skipped_directory_names.push(domain_name.to_string());
318            continue;
319        }
320
321        domain_pairs.push(TlsDomainInspection {
322            domain_name: domain_name.to_string(),
323            pair: inspect_cert_key_pair(&path, &options.cert_filename, &options.key_filename),
324        });
325    }
326
327    domain_pairs.sort_by(|left, right| left.domain_name.cmp(&right.domain_name));
328    skipped_directory_names.sort();
329
330    Ok(TlsDirectoryInspection {
331        directory: dir.to_path_buf(),
332        canonical_directory,
333        root_pair,
334        domain_pairs,
335        skipped_directory_names,
336    })
337}
338
339pub fn load_all_certs_from_directory(
340    options: CertDirectoryLoadOptions,
341) -> io::Result<HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>> {
342    options.validate()?;
343
344    let mut cert_key_pairs = HashMap::new();
345    let dir = options.dir_path.as_path();
346
347    if !dir.exists() || !dir.is_dir() {
348        return Err(certs_error(format!(
349            "The certificate directory does not exist or is not a directory: {}",
350            dir.display()
351        )));
352    }
353
354    let root_cert_path = dir.join(&options.cert_filename);
355    let root_key_path = dir.join(&options.key_filename);
356
357    if root_cert_path.exists() && root_key_path.exists() {
358        debug!("find the root directory certificate: {:?}", root_cert_path);
359        let root_cert_str = root_cert_path
360            .to_str()
361            .ok_or_else(|| certs_error(format!("Invalid UTF-8 in root certificate path: {root_cert_path:?}")))?;
362        let root_key_str = root_key_path
363            .to_str()
364            .ok_or_else(|| certs_error(format!("Invalid UTF-8 in root key path: {root_key_path:?}")))?;
365        match load_cert_key_pair(root_cert_str, root_key_str) {
366            Ok((certs, key)) => {
367                cert_key_pairs.insert("default".to_string(), (certs, key));
368            }
369            Err(e) => {
370                warn!("unable to load root directory certificate: {}", e);
371            }
372        }
373    }
374
375    for entry in fs::read_dir(dir)? {
376        let entry = entry?;
377        let path = entry.path();
378
379        if path.is_dir() {
380            let domain_name: &str = path
381                .file_name()
382                .and_then(|name| name.to_str())
383                .ok_or_else(|| certs_error(format!("invalid domain name directory:{path:?}")))?;
384            if !is_discoverable_cert_domain_dir(domain_name) {
385                debug!("skip internal certificate directory: {:?}", path);
386                continue;
387            }
388
389            let cert_path = path.join(&options.cert_filename);
390            let key_path = path.join(&options.key_filename);
391
392            if cert_path.exists() && key_path.exists() {
393                debug!("find the domain name certificate: {} in {:?}", domain_name, cert_path);
394                let cert_path = match cert_path.to_str() {
395                    Some(path) => path,
396                    None => {
397                        warn!("skip domain certificate load, invalid UTF-8 path: {:?}", cert_path);
398                        continue;
399                    }
400                };
401
402                let key_path = match key_path.to_str() {
403                    Some(path) => path,
404                    None => {
405                        warn!("skip domain key load, invalid UTF-8 path: {:?}", key_path);
406                        continue;
407                    }
408                };
409
410                match load_cert_key_pair(cert_path, key_path) {
411                    Ok((certs, key)) => {
412                        cert_key_pairs.insert(domain_name.to_string(), (certs, key));
413                    }
414                    Err(e) => {
415                        warn!("unable to load the certificate for {} domain name: {}", domain_name, e);
416                    }
417                }
418            }
419        }
420    }
421
422    if cert_key_pairs.is_empty() {
423        return Err(io::Error::new(
424            io::ErrorKind::NotFound,
425            format!("No valid certificate/private key pair found in directory {}", dir.display()),
426        ));
427    }
428
429    Ok(cert_key_pairs)
430}
431
432fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)> {
433    let certs = load_certs(cert_path)?;
434    let key = load_private_key(key_path)?;
435    Ok((certs, key))
436}
437
438fn inspect_cert_key_pair(dir: &Path, cert_filename: &str, key_filename: &str) -> TlsCertPairInspection {
439    let cert_path = dir.join(cert_filename);
440    let key_path = dir.join(key_filename);
441    let cert_exists = cert_path.exists();
442    let key_exists = key_path.exists();
443
444    let status = match (cert_exists, key_exists) {
445        (false, false) => TlsCertPairStatus::MissingBoth,
446        (false, true) => TlsCertPairStatus::MissingCert,
447        (true, false) => TlsCertPairStatus::MissingKey,
448        (true, true) => match cert_key_pair_utf8_paths(&cert_path, &key_path) {
449            Ok((cert_path, key_path)) => match load_cert_key_pair(cert_path, key_path) {
450                Ok(_) => TlsCertPairStatus::Valid,
451                Err(err) => TlsCertPairStatus::Invalid { error: err.to_string() },
452            },
453            Err(err) => TlsCertPairStatus::Invalid { error: err.to_string() },
454        },
455    };
456
457    TlsCertPairInspection {
458        cert_path,
459        key_path,
460        status,
461    }
462}
463
464fn cert_key_pair_utf8_paths<'a>(cert_path: &'a Path, key_path: &'a Path) -> io::Result<(&'a str, &'a str)> {
465    let cert_path = cert_path
466        .to_str()
467        .ok_or_else(|| certs_error(format!("Invalid UTF-8 in certificate path: {cert_path:?}")))?;
468    let key_path = key_path
469        .to_str()
470        .ok_or_else(|| certs_error(format!("Invalid UTF-8 in key path: {key_path:?}")))?;
471    Ok((cert_path, key_path))
472}
473
474pub fn create_multi_cert_resolver(
475    cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
476) -> io::Result<impl ResolvesServerCert> {
477    #[derive(Debug)]
478    struct MultiCertResolver {
479        cert_resolver: ResolvesServerCertUsingSni,
480        default_cert: Option<Arc<CertifiedKey>>,
481    }
482
483    impl ResolvesServerCert for MultiCertResolver {
484        fn resolve(&self, client_hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
485            if let Some(cert) = self.cert_resolver.resolve(client_hello) {
486                return Some(cert);
487            }
488
489            self.default_cert.clone()
490        }
491    }
492
493    let mut resolver = ResolvesServerCertUsingSni::new();
494    let mut default_cert = None;
495
496    for (domain, (certs, key)) in cert_key_pairs {
497        let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
498            .map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?;
499
500        let certified_key = CertifiedKey::new(certs, signing_key);
501        if domain == "default" {
502            default_cert = Some(Arc::new(certified_key.clone()));
503        } else {
504            resolver
505                .add(&domain, certified_key)
506                .map_err(|e| certs_error(format!("failed to add a domain name certificate:{domain},err: {e:?}")))?;
507        }
508    }
509
510    Ok(MultiCertResolver {
511        cert_resolver: resolver,
512        default_cert,
513    })
514}
515
516#[cfg(test)]
517mod tests {
518    use super::*;
519    use std::fs;
520    use std::io::ErrorKind;
521    use std::path::PathBuf;
522    use tempfile::TempDir;
523
524    fn default_load_options(path: impl Into<PathBuf>) -> CertDirectoryLoadOptions {
525        CertDirectoryLoadOptions::builder(path, "rustfs_cert.pem", "rustfs_key.pem").build()
526    }
527
528    fn write_test_cert_pair(dir: &std::path::Path) {
529        let rcgen::CertifiedKey { cert, signing_key } =
530            rcgen::generate_simple_self_signed(vec!["example.com".to_string()]).expect("cert should generate");
531        fs::write(dir.join("rustfs_cert.pem"), cert.pem()).expect("cert should write");
532        fs::write(dir.join("rustfs_key.pem"), signing_key.serialize_pem()).expect("key should write");
533    }
534
535    #[test]
536    fn test_certs_error_function() {
537        let error_msg = "Test error message";
538        let error = certs_error(error_msg.to_string());
539
540        assert_eq!(error.kind(), ErrorKind::Other);
541        assert_eq!(error.to_string(), error_msg);
542    }
543
544    #[test]
545    fn test_load_certs_file_not_found() {
546        let result = load_certs("non_existent_file.pem");
547        assert!(result.is_err());
548
549        let error = result.expect_err("missing cert should error");
550        assert_eq!(error.kind(), ErrorKind::Other);
551        assert!(error.to_string().contains("failed to open"));
552    }
553
554    #[test]
555    fn test_load_private_key_file_not_found() {
556        let result = load_private_key("non_existent_key.pem");
557        assert!(result.is_err());
558
559        let error = result.expect_err("missing key should error");
560        assert_eq!(error.kind(), ErrorKind::Other);
561        assert!(error.to_string().contains("failed to open"));
562    }
563
564    #[test]
565    fn test_load_all_certs_from_directory_empty() {
566        let temp_dir = TempDir::new().expect("tempdir should create");
567        let result = load_all_certs_from_directory(default_load_options(temp_dir.path()));
568        assert!(result.is_err());
569        let error = result.expect_err("empty directory should error");
570        assert_eq!(error.kind(), ErrorKind::NotFound);
571        assert!(error.to_string().contains("No valid certificate/private key pair found"));
572    }
573
574    #[test]
575    fn test_load_all_certs_skips_kubernetes_secret_projection_dirs() {
576        let temp_dir = TempDir::new().expect("tempdir should create");
577        write_test_cert_pair(temp_dir.path());
578
579        let domain_dir = temp_dir.path().join("example.com");
580        fs::create_dir(&domain_dir).expect("domain dir should create");
581        write_test_cert_pair(&domain_dir);
582
583        for internal_dir_name in ["..data", "..2026_04_28_18_33_53.4209048473"] {
584            let internal_dir = temp_dir.path().join(internal_dir_name);
585            fs::create_dir(&internal_dir).expect("internal dir should create");
586            write_test_cert_pair(&internal_dir);
587        }
588
589        let certs = load_all_certs_from_directory(default_load_options(temp_dir.path())).expect("certs should load");
590        assert!(certs.contains_key("default"));
591        assert!(certs.contains_key("example.com"));
592        assert!(!certs.contains_key("..data"));
593        assert_eq!(certs.len(), 2);
594    }
595
596    #[test]
597    fn test_inspect_cert_directory_reports_valid_root_and_domain_pairs() {
598        let temp_dir = TempDir::new().expect("tempdir should create");
599        write_test_cert_pair(temp_dir.path());
600
601        let domain_dir = temp_dir.path().join("example.com");
602        fs::create_dir(&domain_dir).expect("domain dir should create");
603        write_test_cert_pair(&domain_dir);
604
605        let inspection = inspect_cert_directory(default_load_options(temp_dir.path())).expect("inspection should succeed");
606        assert!(inspection.has_valid_root_pair());
607        assert_eq!(inspection.valid_domain_names(), vec!["example.com"]);
608        assert_eq!(inspection.domain_pairs.len(), 1);
609        assert!(inspection.domain_pairs[0].pair.is_valid());
610    }
611
612    #[test]
613    fn test_inspect_cert_directory_reports_invalid_and_missing_pairs() {
614        let temp_dir = TempDir::new().expect("tempdir should create");
615        fs::write(temp_dir.path().join("rustfs_cert.pem"), "invalid certificate").expect("invalid cert should write");
616        fs::write(temp_dir.path().join("rustfs_key.pem"), "invalid key").expect("invalid key should write");
617
618        let domain_dir = temp_dir.path().join("broken.example.com");
619        fs::create_dir(&domain_dir).expect("domain dir should create");
620        fs::write(domain_dir.join("rustfs_cert.pem"), "invalid certificate").expect("invalid cert should write");
621
622        let inspection = inspect_cert_directory(default_load_options(temp_dir.path())).expect("inspection should succeed");
623        assert!(matches!(inspection.root_pair.status, TlsCertPairStatus::Invalid { .. }));
624        assert_eq!(inspection.domain_pairs.len(), 1);
625        assert_eq!(inspection.domain_pairs[0].pair.status, TlsCertPairStatus::MissingKey);
626    }
627}
rustfs_tls_runtime/certs.rs

rustfs_tls_runtime/
certs.rs
