object_store/gcp/
builder.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::client::{HttpConnector, TokenCredentialProvider, http_connector};
19use crate::config::ConfigValue;
20use crate::gcp::client::{GoogleCloudStorageClient, GoogleCloudStorageConfig};
21use crate::gcp::credential::{
22    ApplicationDefaultCredentials, DEFAULT_GCS_BASE_URL, InstanceCredentialProvider,
23    ServiceAccountCredentials,
24};
25use crate::gcp::{
26    GcpCredential, GcpCredentialProvider, GcpSigningCredential, GcpSigningCredentialProvider,
27    GoogleCloudStorage, STORE, credential,
28};
29use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
30use serde::{Deserialize, Serialize};
31use std::str::FromStr;
32use std::sync::Arc;
33use std::time::Duration;
34use url::Url;
35
36use super::credential::{AuthorizedUserSigningCredentials, InstanceSigningCredentialProvider};
37
38const TOKEN_MIN_TTL: Duration = Duration::from_secs(4 * 60);
39
40#[derive(Debug, thiserror::Error)]
41enum Error {
42    #[error("Missing bucket name")]
43    MissingBucketName {},
44
45    #[error("One of service account path or service account key may be provided.")]
46    ServiceAccountPathAndKeyProvided,
47
48    #[error("Unable parse source url. Url: {}, Error: {}", url, source)]
49    UnableToParseUrl {
50        source: url::ParseError,
51        url: String,
52    },
53
54    #[error(
55        "Unknown url scheme cannot be parsed into storage location: {}",
56        scheme
57    )]
58    UnknownUrlScheme { scheme: String },
59
60    #[error("URL did not match any known pattern for scheme: {}", url)]
61    UrlNotRecognised { url: String },
62
63    #[error("Configuration key: '{}' is not known.", key)]
64    UnknownConfigurationKey { key: String },
65
66    #[error("GCP credential error: {}", source)]
67    Credential { source: credential::Error },
68}
69
70impl From<Error> for crate::Error {
71    fn from(err: Error) -> Self {
72        match err {
73            Error::UnknownConfigurationKey { key } => {
74                Self::UnknownConfigurationKey { store: STORE, key }
75            }
76            _ => Self::Generic {
77                store: STORE,
78                source: Box::new(err),
79            },
80        }
81    }
82}
83
84/// Configure a connection to Google Cloud Storage.
85///
86/// If no credentials are explicitly provided, they will be sourced
87/// from the environment as documented [here](https://cloud.google.com/docs/authentication/application-default-credentials).
88///
89/// # Example
90/// ```
91/// # let BUCKET_NAME = "foo";
92/// # use object_store::gcp::GoogleCloudStorageBuilder;
93/// let gcs = GoogleCloudStorageBuilder::from_env().with_bucket_name(BUCKET_NAME).build();
94/// ```
95#[derive(Debug, Clone)]
96pub struct GoogleCloudStorageBuilder {
97    /// Bucket name
98    bucket_name: Option<String>,
99    /// Url
100    url: Option<String>,
101    /// Base URL
102    base_url: Option<String>,
103    /// Path to the service account file
104    service_account_path: Option<String>,
105    /// The serialized service account key
106    service_account_key: Option<String>,
107    /// Path to the application credentials file.
108    application_credentials_path: Option<String>,
109    /// Retry config
110    retry_config: RetryConfig,
111    /// Client options
112    client_options: ClientOptions,
113    /// Credentials
114    credentials: Option<GcpCredentialProvider>,
115    /// Skip signing requests
116    skip_signature: ConfigValue<bool>,
117    /// Credentials for sign url
118    signing_credentials: Option<GcpSigningCredentialProvider>,
119    /// The [`HttpConnector`] to use
120    http_connector: Option<Arc<dyn HttpConnector>>,
121}
122
123/// Configuration keys for [`GoogleCloudStorageBuilder`]
124///
125/// Configuration via keys can be done via [`GoogleCloudStorageBuilder::with_config`]
126///
127/// # Example
128/// ```
129/// # use object_store::gcp::{GoogleCloudStorageBuilder, GoogleConfigKey};
130/// let builder = GoogleCloudStorageBuilder::new()
131///     .with_config("google_service_account".parse().unwrap(), "my-service-account")
132///     .with_config(GoogleConfigKey::Bucket, "my-bucket");
133/// ```
134#[derive(PartialEq, Eq, Hash, Clone, Debug, Copy, Serialize, Deserialize)]
135#[non_exhaustive]
136pub enum GoogleConfigKey {
137    /// Path to the service account file
138    ///
139    /// Supported keys:
140    /// - `google_service_account`
141    /// - `service_account`
142    /// - `google_service_account_path`
143    /// - `service_account_path`
144    ServiceAccount,
145
146    /// The serialized service account key.
147    ///
148    /// Supported keys:
149    /// - `google_service_account_key`
150    /// - `service_account_key`
151    ServiceAccountKey,
152
153    /// Bucket name
154    ///
155    /// See [`GoogleCloudStorageBuilder::with_bucket_name`] for details.
156    ///
157    /// Supported keys:
158    /// - `google_bucket`
159    /// - `google_bucket_name`
160    /// - `bucket`
161    /// - `bucket_name`
162    Bucket,
163
164    /// Base URL
165    ///
166    /// See [`GoogleCloudStorageBuilder::with_base_url`] for details.
167    ///
168    /// Supported keys:
169    /// - `google_base_url`
170    /// - `base_url`
171    BaseUrl,
172
173    /// Application credentials path
174    ///
175    /// See [`GoogleCloudStorageBuilder::with_application_credentials`].
176    ///
177    /// Supported keys:
178    /// - `google_application_credentials`
179    /// - `application_credentials`
180    ApplicationCredentials,
181
182    /// Skip signing request
183    ///
184    /// Supported keys:
185    /// - `google_skip_signature`
186    /// - `skip_signature`
187    SkipSignature,
188
189    /// Client options
190    Client(ClientConfigKey),
191}
192
193impl AsRef<str> for GoogleConfigKey {
194    fn as_ref(&self) -> &str {
195        match self {
196            Self::ServiceAccount => "google_service_account",
197            Self::ServiceAccountKey => "google_service_account_key",
198            Self::Bucket => "google_bucket",
199            Self::BaseUrl => "google_base_url",
200            Self::ApplicationCredentials => "google_application_credentials",
201            Self::SkipSignature => "google_skip_signature",
202            Self::Client(key) => key.as_ref(),
203        }
204    }
205}
206
207impl FromStr for GoogleConfigKey {
208    type Err = crate::Error;
209
210    fn from_str(s: &str) -> Result<Self, Self::Err> {
211        match s {
212            "google_service_account"
213            | "service_account"
214            | "google_service_account_path"
215            | "service_account_path" => Ok(Self::ServiceAccount),
216            "google_service_account_key" | "service_account_key" => Ok(Self::ServiceAccountKey),
217            "google_bucket" | "google_bucket_name" | "bucket" | "bucket_name" => Ok(Self::Bucket),
218            "google_base_url" | "base_url" => Ok(Self::BaseUrl),
219            "google_application_credentials" | "application_credentials" => {
220                Ok(Self::ApplicationCredentials)
221            }
222            "google_skip_signature" | "skip_signature" => Ok(Self::SkipSignature),
223            _ => match s.strip_prefix("google_").unwrap_or(s).parse() {
224                Ok(key) => Ok(Self::Client(key)),
225                Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
226            },
227        }
228    }
229}
230
231impl Default for GoogleCloudStorageBuilder {
232    fn default() -> Self {
233        Self {
234            bucket_name: None,
235            service_account_path: None,
236            service_account_key: None,
237            application_credentials_path: None,
238            retry_config: Default::default(),
239            client_options: ClientOptions::new().with_allow_http(true),
240            url: None,
241            base_url: None,
242            credentials: None,
243            skip_signature: Default::default(),
244            signing_credentials: None,
245            http_connector: None,
246        }
247    }
248}
249
250impl GoogleCloudStorageBuilder {
251    /// Create a new [`GoogleCloudStorageBuilder`] with default values.
252    pub fn new() -> Self {
253        Default::default()
254    }
255
256    /// Create an instance of [`GoogleCloudStorageBuilder`] with values pre-populated from environment variables.
257    ///
258    /// Variables extracted from environment:
259    /// * GOOGLE_SERVICE_ACCOUNT: location of service account file
260    /// * GOOGLE_SERVICE_ACCOUNT_PATH: (alias) location of service account file
261    /// * SERVICE_ACCOUNT: (alias) location of service account file
262    /// * GOOGLE_SERVICE_ACCOUNT_KEY: JSON serialized service account key
263    /// * GOOGLE_BUCKET: bucket name
264    /// * GOOGLE_BUCKET_NAME: (alias) bucket name
265    ///
266    /// # Example
267    /// ```
268    /// use object_store::gcp::GoogleCloudStorageBuilder;
269    ///
270    /// let gcs = GoogleCloudStorageBuilder::from_env()
271    ///     .with_bucket_name("foo")
272    ///     .build();
273    /// ```
274    pub fn from_env() -> Self {
275        let mut builder = Self::default();
276
277        if let Ok(service_account_path) = std::env::var("SERVICE_ACCOUNT") {
278            builder.service_account_path = Some(service_account_path);
279        }
280
281        for (os_key, os_value) in std::env::vars_os() {
282            if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
283                if key.starts_with("GOOGLE_") {
284                    if let Ok(config_key) = key.to_ascii_lowercase().parse() {
285                        builder = builder.with_config(config_key, value);
286                    }
287                }
288            }
289        }
290
291        builder
292    }
293
294    /// Parse available connection info form a well-known storage URL.
295    ///
296    /// The supported url schemes are:
297    ///
298    /// - `gs://<bucket>/<path>`
299    ///
300    /// Note: Settings derived from the URL will override any others set on this builder
301    ///
302    /// # Example
303    /// ```
304    /// use object_store::gcp::GoogleCloudStorageBuilder;
305    ///
306    /// let gcs = GoogleCloudStorageBuilder::from_env()
307    ///     .with_url("gs://bucket/path")
308    ///     .build();
309    /// ```
310    pub fn with_url(mut self, url: impl Into<String>) -> Self {
311        self.url = Some(url.into());
312        self
313    }
314
315    /// Set an option on the builder via a key - value pair.
316    pub fn with_config(mut self, key: GoogleConfigKey, value: impl Into<String>) -> Self {
317        match key {
318            GoogleConfigKey::ServiceAccount => self.service_account_path = Some(value.into()),
319            GoogleConfigKey::ServiceAccountKey => self.service_account_key = Some(value.into()),
320            GoogleConfigKey::Bucket => self.bucket_name = Some(value.into()),
321            GoogleConfigKey::BaseUrl => self.base_url = Some(value.into()),
322            GoogleConfigKey::ApplicationCredentials => {
323                self.application_credentials_path = Some(value.into())
324            }
325            GoogleConfigKey::SkipSignature => self.skip_signature.parse(value),
326            GoogleConfigKey::Client(key) => {
327                self.client_options = self.client_options.with_config(key, value)
328            }
329        };
330        self
331    }
332
333    /// Get config value via a [`GoogleConfigKey`].
334    ///
335    /// # Example
336    /// ```
337    /// use object_store::gcp::{GoogleCloudStorageBuilder, GoogleConfigKey};
338    ///
339    /// let builder = GoogleCloudStorageBuilder::from_env()
340    ///     .with_service_account_key("foo");
341    /// let service_account_key = builder.get_config_value(&GoogleConfigKey::ServiceAccountKey).unwrap_or_default();
342    /// assert_eq!("foo", &service_account_key);
343    /// ```
344    pub fn get_config_value(&self, key: &GoogleConfigKey) -> Option<String> {
345        match key {
346            GoogleConfigKey::ServiceAccount => self.service_account_path.clone(),
347            GoogleConfigKey::ServiceAccountKey => self.service_account_key.clone(),
348            GoogleConfigKey::Bucket => self.bucket_name.clone(),
349            GoogleConfigKey::BaseUrl => self.base_url.clone(),
350            GoogleConfigKey::ApplicationCredentials => self.application_credentials_path.clone(),
351            GoogleConfigKey::SkipSignature => Some(self.skip_signature.to_string()),
352            GoogleConfigKey::Client(key) => self.client_options.get_config_value(key),
353        }
354    }
355
356    /// Sets properties on this builder based on a URL
357    ///
358    /// This is a separate member function to allow fallible computation to
359    /// be deferred until [`Self::build`] which in turn allows deriving [`Clone`]
360    fn parse_url(&mut self, url: &str) -> Result<()> {
361        let parsed = Url::parse(url).map_err(|source| Error::UnableToParseUrl {
362            source,
363            url: url.to_string(),
364        })?;
365
366        let host = parsed.host_str().ok_or_else(|| Error::UrlNotRecognised {
367            url: url.to_string(),
368        })?;
369
370        match parsed.scheme() {
371            "gs" => self.bucket_name = Some(host.to_string()),
372            scheme => {
373                let scheme = scheme.to_string();
374                return Err(Error::UnknownUrlScheme { scheme }.into());
375            }
376        }
377        Ok(())
378    }
379
380    /// Set the bucket name (required)
381    pub fn with_bucket_name(mut self, bucket_name: impl Into<String>) -> Self {
382        self.bucket_name = Some(bucket_name.into());
383        self
384    }
385
386    /// Sets the base URL for communicating with GCS.
387    ///
388    /// If not explicitly set, it will be:
389    /// 1. Derived from the service account credentials, if provided
390    /// 2. Otherwise, uses the default GCS endpoint
391    ///
392    /// # Example
393    /// ```
394    /// use object_store::gcp::GoogleCloudStorageBuilder;
395    ///
396    /// let gcs = GoogleCloudStorageBuilder::from_env()
397    ///     .with_base_url("https://localhost:4443")
398    ///     .build();
399    /// ```
400    pub fn with_base_url(mut self, base_url: &str) -> Self {
401        self.base_url = Some(base_url.into());
402        self
403    }
404
405    /// Set the path to the service account file.
406    ///
407    /// This or [`GoogleCloudStorageBuilder::with_service_account_key`] must be
408    /// set.
409    ///
410    /// Example `"/tmp/gcs.json"`.
411    ///
412    /// Example contents of `gcs.json`:
413    ///
414    /// ```json
415    /// {
416    ///    "gcs_base_url": "https://localhost:4443",
417    ///    "disable_oauth": true,
418    ///    "client_email": "",
419    ///    "private_key": ""
420    /// }
421    /// ```
422    pub fn with_service_account_path(mut self, service_account_path: impl Into<String>) -> Self {
423        self.service_account_path = Some(service_account_path.into());
424        self
425    }
426
427    /// Set the service account key. The service account must be in the JSON
428    /// format.
429    ///
430    /// This or [`GoogleCloudStorageBuilder::with_service_account_path`] must be
431    /// set.
432    pub fn with_service_account_key(mut self, service_account: impl Into<String>) -> Self {
433        self.service_account_key = Some(service_account.into());
434        self
435    }
436
437    /// Set the path to the application credentials file.
438    ///
439    /// <https://cloud.google.com/docs/authentication/provide-credentials-adc>
440    pub fn with_application_credentials(
441        mut self,
442        application_credentials_path: impl Into<String>,
443    ) -> Self {
444        self.application_credentials_path = Some(application_credentials_path.into());
445        self
446    }
447
448    /// If enabled, [`GoogleCloudStorage`] will not fetch credentials and will not sign requests.
449    ///
450    /// This can be useful when interacting with public GCS buckets that deny authorized requests.
451    pub fn with_skip_signature(mut self, skip_signature: bool) -> Self {
452        self.skip_signature = skip_signature.into();
453        self
454    }
455
456    /// Set the credential provider overriding any other options
457    pub fn with_credentials(mut self, credentials: GcpCredentialProvider) -> Self {
458        self.credentials = Some(credentials);
459        self
460    }
461
462    /// Set the retry configuration
463    pub fn with_retry(mut self, retry_config: RetryConfig) -> Self {
464        self.retry_config = retry_config;
465        self
466    }
467
468    /// Set the proxy_url to be used by the underlying client
469    pub fn with_proxy_url(mut self, proxy_url: impl Into<String>) -> Self {
470        self.client_options = self.client_options.with_proxy_url(proxy_url);
471        self
472    }
473
474    /// Set a trusted proxy CA certificate
475    pub fn with_proxy_ca_certificate(mut self, proxy_ca_certificate: impl Into<String>) -> Self {
476        self.client_options = self
477            .client_options
478            .with_proxy_ca_certificate(proxy_ca_certificate);
479        self
480    }
481
482    /// Set a list of hosts to exclude from proxy connections
483    pub fn with_proxy_excludes(mut self, proxy_excludes: impl Into<String>) -> Self {
484        self.client_options = self.client_options.with_proxy_excludes(proxy_excludes);
485        self
486    }
487
488    /// Sets the client options, overriding any already set
489    pub fn with_client_options(mut self, options: ClientOptions) -> Self {
490        self.client_options = options;
491        self
492    }
493
494    /// The [`HttpConnector`] to use
495    ///
496    /// On non-WASM32 platforms uses [`reqwest`] by default, on WASM32 platforms must be provided
497    pub fn with_http_connector<C: HttpConnector>(mut self, connector: C) -> Self {
498        self.http_connector = Some(Arc::new(connector));
499        self
500    }
501
502    /// Configure a connection to Google Cloud Storage, returning a
503    /// new [`GoogleCloudStorage`] and consuming `self`
504    pub fn build(mut self) -> Result<GoogleCloudStorage> {
505        if let Some(url) = self.url.take() {
506            self.parse_url(&url)?;
507        }
508
509        let bucket_name = self.bucket_name.ok_or(Error::MissingBucketName {})?;
510
511        let http = http_connector(self.http_connector)?;
512
513        // First try to initialize from the service account information.
514        let service_account_credentials =
515            match (self.service_account_path, self.service_account_key) {
516                (Some(path), None) => Some(
517                    ServiceAccountCredentials::from_file(path)
518                        .map_err(|source| Error::Credential { source })?,
519                ),
520                (None, Some(key)) => Some(
521                    ServiceAccountCredentials::from_key(&key)
522                        .map_err(|source| Error::Credential { source })?,
523                ),
524                (None, None) => None,
525                (Some(_), Some(_)) => return Err(Error::ServiceAccountPathAndKeyProvided.into()),
526            };
527
528        // Then try to initialize from the application credentials file, or the environment.
529        // Only attempt to read ADC if no explicit credentials were provided
530        let application_default_credentials =
531            if service_account_credentials.is_none() && self.credentials.is_none() {
532                // No explicit credentials, so try ADC and propagate errors
533                ApplicationDefaultCredentials::read(self.application_credentials_path.as_deref())?
534            } else {
535                // Explicit credentials provided, skip ADC reading entirely
536                None
537            };
538
539        let disable_oauth = service_account_credentials
540            .as_ref()
541            .map(|c| c.disable_oauth)
542            .unwrap_or(false);
543
544        let gcs_base_url: String = self
545            .base_url
546            .or_else(|| {
547                service_account_credentials
548                    .as_ref()
549                    .and_then(|c| c.gcs_base_url.clone())
550            })
551            .unwrap_or_else(|| DEFAULT_GCS_BASE_URL.to_string());
552
553        let credentials = if let Some(credentials) = self.credentials {
554            credentials
555        } else if disable_oauth {
556            Arc::new(StaticCredentialProvider::new(GcpCredential {
557                bearer: "".to_string(),
558            })) as _
559        } else if let Some(credentials) = service_account_credentials.clone() {
560            Arc::new(TokenCredentialProvider::new(
561                credentials.token_provider()?,
562                http.connect(&self.client_options)?,
563                self.retry_config.clone(),
564            )) as _
565        } else if let Some(credentials) = application_default_credentials.clone() {
566            match credentials {
567                ApplicationDefaultCredentials::AuthorizedUser(token) => Arc::new(
568                    TokenCredentialProvider::new(
569                        token,
570                        http.connect(&self.client_options)?,
571                        self.retry_config.clone(),
572                    )
573                    .with_min_ttl(TOKEN_MIN_TTL),
574                ) as _,
575                ApplicationDefaultCredentials::ServiceAccount(token) => {
576                    Arc::new(TokenCredentialProvider::new(
577                        token.token_provider()?,
578                        http.connect(&self.client_options)?,
579                        self.retry_config.clone(),
580                    )) as _
581                }
582            }
583        } else {
584            Arc::new(
585                TokenCredentialProvider::new(
586                    InstanceCredentialProvider::default(),
587                    http.connect(&self.client_options.metadata_options())?,
588                    self.retry_config.clone(),
589                )
590                .with_min_ttl(TOKEN_MIN_TTL),
591            ) as _
592        };
593
594        let signing_credentials = if let Some(signing_credentials) = self.signing_credentials {
595            signing_credentials
596        } else if disable_oauth {
597            Arc::new(StaticCredentialProvider::new(GcpSigningCredential {
598                email: "".to_string(),
599                private_key: None,
600            })) as _
601        } else if let Some(credentials) = service_account_credentials.clone() {
602            credentials.signing_credentials()?
603        } else if let Some(credentials) = application_default_credentials.clone() {
604            match credentials {
605                ApplicationDefaultCredentials::AuthorizedUser(token) => {
606                    Arc::new(TokenCredentialProvider::new(
607                        AuthorizedUserSigningCredentials::from(token)?,
608                        http.connect(&self.client_options)?,
609                        self.retry_config.clone(),
610                    )) as _
611                }
612                ApplicationDefaultCredentials::ServiceAccount(token) => {
613                    token.signing_credentials()?
614                }
615            }
616        } else {
617            Arc::new(TokenCredentialProvider::new(
618                InstanceSigningCredentialProvider::default(),
619                http.connect(&self.client_options.metadata_options())?,
620                self.retry_config.clone(),
621            )) as _
622        };
623
624        let config = GoogleCloudStorageConfig {
625            base_url: gcs_base_url,
626            credentials,
627            signing_credentials,
628            bucket_name,
629            retry_config: self.retry_config,
630            client_options: self.client_options,
631            skip_signature: self.skip_signature.get()?,
632        };
633
634        let http_client = http.connect(&config.client_options)?;
635        Ok(GoogleCloudStorage {
636            client: Arc::new(GoogleCloudStorageClient::new(config, http_client)?),
637        })
638    }
639}
640
641#[cfg(test)]
642mod tests {
643    use super::*;
644    use std::collections::HashMap;
645    use std::io::Write;
646    use tempfile::NamedTempFile;
647
648    const FAKE_KEY: &str = r#"{"private_key": "private_key", "private_key_id": "private_key_id", "client_email":"client_email", "disable_oauth":true}"#;
649    const FAKE_KEY_WITH_BASE_URL: &str = r#"{"private_key": "private_key", "private_key_id": "private_key_id", "client_email":"client_email", "disable_oauth":true, "gcs_base_url": "https://base-url-from-credentials:4443"}"#;
650
651    #[test]
652    fn gcs_test_service_account_key_and_path() {
653        let mut tfile = NamedTempFile::new().unwrap();
654        write!(tfile, "{FAKE_KEY}").unwrap();
655        let _ = GoogleCloudStorageBuilder::new()
656            .with_service_account_key(FAKE_KEY)
657            .with_service_account_path(tfile.path().to_str().unwrap())
658            .with_bucket_name("foo")
659            .build()
660            .unwrap_err();
661    }
662
663    #[test]
664    fn gcs_test_config_from_map() {
665        let google_service_account = "object_store:fake_service_account".to_string();
666        let google_bucket_name = "object_store:fake_bucket".to_string();
667        let options = HashMap::from([
668            ("google_service_account", google_service_account.clone()),
669            ("google_bucket_name", google_bucket_name.clone()),
670        ]);
671
672        let builder = options
673            .iter()
674            .fold(GoogleCloudStorageBuilder::new(), |builder, (key, value)| {
675                builder.with_config(key.parse().unwrap(), value)
676            });
677
678        assert_eq!(
679            builder.service_account_path.unwrap(),
680            google_service_account.as_str()
681        );
682        assert_eq!(builder.bucket_name.unwrap(), google_bucket_name.as_str());
683    }
684
685    #[test]
686    fn gcs_test_config_aliases() {
687        // Service account path
688        for alias in [
689            "google_service_account",
690            "service_account",
691            "google_service_account_path",
692            "service_account_path",
693        ] {
694            let builder = GoogleCloudStorageBuilder::new()
695                .with_config(alias.parse().unwrap(), "/fake/path.json");
696            assert_eq!("/fake/path.json", builder.service_account_path.unwrap());
697        }
698
699        // Service account key
700        for alias in ["google_service_account_key", "service_account_key"] {
701            let builder =
702                GoogleCloudStorageBuilder::new().with_config(alias.parse().unwrap(), FAKE_KEY);
703            assert_eq!(FAKE_KEY, builder.service_account_key.unwrap());
704        }
705
706        // Bucket name
707        for alias in [
708            "google_bucket",
709            "google_bucket_name",
710            "bucket",
711            "bucket_name",
712        ] {
713            let builder =
714                GoogleCloudStorageBuilder::new().with_config(alias.parse().unwrap(), "fake_bucket");
715            assert_eq!("fake_bucket", builder.bucket_name.unwrap());
716        }
717    }
718
719    #[tokio::test]
720    async fn gcs_test_proxy_url() {
721        let mut tfile = NamedTempFile::new().unwrap();
722        write!(tfile, "{FAKE_KEY}").unwrap();
723        let service_account_path = tfile.path();
724        let gcs = GoogleCloudStorageBuilder::new()
725            .with_service_account_path(service_account_path.to_str().unwrap())
726            .with_bucket_name("foo")
727            .with_proxy_url("https://example.com")
728            .build();
729        assert!(gcs.is_ok());
730
731        let err = GoogleCloudStorageBuilder::new()
732            .with_service_account_path(service_account_path.to_str().unwrap())
733            .with_bucket_name("foo")
734            // use invalid url
735            .with_proxy_url("dxx:ddd\\example.com")
736            .build()
737            .unwrap_err()
738            .to_string();
739
740        assert_eq!("Generic HTTP client error: builder error", err);
741    }
742
743    #[test]
744    fn gcs_test_urls() {
745        let mut builder = GoogleCloudStorageBuilder::new();
746        builder.parse_url("gs://bucket/path").unwrap();
747        assert_eq!(builder.bucket_name.as_deref(), Some("bucket"));
748
749        builder.parse_url("gs://bucket.mydomain/path").unwrap();
750        assert_eq!(builder.bucket_name.as_deref(), Some("bucket.mydomain"));
751
752        builder.parse_url("mailto://bucket/path").unwrap_err();
753    }
754
755    #[test]
756    fn gcs_test_service_account_key_only() {
757        let _ = GoogleCloudStorageBuilder::new()
758            .with_service_account_key(FAKE_KEY)
759            .with_bucket_name("foo")
760            .build()
761            .unwrap();
762    }
763
764    #[test]
765    fn gcs_test_with_base_url() {
766        let no_base_url = GoogleCloudStorageBuilder::new()
767            .with_bucket_name("foo")
768            .build()
769            .unwrap();
770        assert_eq!(no_base_url.client.config().base_url, DEFAULT_GCS_BASE_URL);
771
772        let explicit_override = GoogleCloudStorageBuilder::new()
773            .with_bucket_name("foo")
774            .with_base_url("https://explicitly-overridden:4443")
775            .build()
776            .unwrap();
777        assert_eq!(
778            explicit_override.client.config().base_url,
779            "https://explicitly-overridden:4443"
780        );
781
782        let url_in_credentials = GoogleCloudStorageBuilder::new()
783            .with_bucket_name("foo")
784            .with_service_account_key(FAKE_KEY_WITH_BASE_URL)
785            .build()
786            .unwrap();
787        assert_eq!(
788            url_in_credentials.client.config().base_url,
789            "https://base-url-from-credentials:4443"
790        );
791
792        let explicit_override_and_credentials = GoogleCloudStorageBuilder::new()
793            .with_bucket_name("foo")
794            .with_base_url("https://explicitly-overridden:4443") // this should take precedence
795            .with_service_account_key(FAKE_KEY_WITH_BASE_URL)
796            .build()
797            .unwrap();
798        assert_eq!(
799            explicit_override_and_credentials.client.config().base_url,
800            "https://explicitly-overridden:4443"
801        );
802    }
803
804    #[test]
805    fn gcs_test_config_get_value() {
806        let google_service_account = "object_store:fake_service_account".to_string();
807        let google_bucket_name = "object_store:fake_bucket".to_string();
808        let builder = GoogleCloudStorageBuilder::new()
809            .with_config(GoogleConfigKey::ServiceAccount, &google_service_account)
810            .with_config(GoogleConfigKey::Bucket, &google_bucket_name);
811
812        assert_eq!(
813            builder
814                .get_config_value(&GoogleConfigKey::ServiceAccount)
815                .unwrap(),
816            google_service_account
817        );
818        assert_eq!(
819            builder.get_config_value(&GoogleConfigKey::Bucket).unwrap(),
820            google_bucket_name
821        );
822    }
823
824    #[test]
825    fn gcp_test_client_opts() {
826        let key = "GOOGLE_PROXY_URL";
827        if let Ok(config_key) = key.to_ascii_lowercase().parse() {
828            assert_eq!(
829                GoogleConfigKey::Client(ClientConfigKey::ProxyUrl),
830                config_key
831            );
832        } else {
833            panic!("{key} not propagated as ClientConfigKey");
834        }
835    }
836
837    #[test]
838    fn gcs_test_explicit_creds_skip_invalid_adc() {
839        // Create a valid service account key file
840        let mut valid_key_file = NamedTempFile::new().unwrap();
841        write!(valid_key_file, "{FAKE_KEY}").unwrap();
842
843        // Create invalid ADC file with unsupported credential type
844        let mut invalid_adc_file = NamedTempFile::new().unwrap();
845        invalid_adc_file
846            .write_all(br#"{"type": "external_account_authorized_user", "audience": "test"}"#)
847            .unwrap();
848
849        // Build should succeed because explicit credentials are provided
850        // and ADC errors should be ignored
851        let result = GoogleCloudStorageBuilder::new()
852            .with_service_account_path(valid_key_file.path().to_str().unwrap())
853            .with_application_credentials(invalid_adc_file.path().to_str().unwrap())
854            .with_bucket_name("test-bucket")
855            .build();
856
857        // Should succeed - ADC errors should be ignored when explicit creds provided
858        assert!(
859            result.is_ok(),
860            "Build should succeed with explicit credentials despite invalid ADC: {:?}",
861            result.err()
862        );
863    }
864
865    #[test]
866    fn gcs_test_explicit_creds_with_service_account_key_skip_invalid_adc() {
867        // Create invalid ADC file with unsupported credential type
868        let mut invalid_adc_file = NamedTempFile::new().unwrap();
869        invalid_adc_file
870            .write_all(br#"{"type": "external_account_authorized_user", "audience": "test"}"#)
871            .unwrap();
872
873        // Build should succeed with service account key (not path)
874        let result = GoogleCloudStorageBuilder::new()
875            .with_service_account_key(FAKE_KEY)
876            .with_application_credentials(invalid_adc_file.path().to_str().unwrap())
877            .with_bucket_name("test-bucket")
878            .build();
879
880        // Should succeed - ADC errors should be ignored when explicit creds provided
881        assert!(
882            result.is_ok(),
883            "Build should succeed with service account key despite invalid ADC: {:?}",
884            result.err()
885        );
886    }
887
888    #[test]
889    fn gcs_test_adc_error_propagated_without_explicit_creds() {
890        // Create invalid ADC file with unsupported credential type
891        let mut invalid_adc_file = NamedTempFile::new().unwrap();
892        invalid_adc_file
893            .write_all(br#"{"type": "external_account_authorized_user", "audience": "test"}"#)
894            .unwrap();
895
896        // Build should fail because no explicit credentials and ADC is invalid
897        let result = GoogleCloudStorageBuilder::new()
898            .with_application_credentials(invalid_adc_file.path().to_str().unwrap())
899            .with_bucket_name("test-bucket")
900            .build();
901
902        // Should fail - ADC errors should be propagated when no explicit creds
903        assert!(
904            result.is_err(),
905            "Build should fail without explicit credentials and invalid ADC"
906        );
907        let err_msg = result.unwrap_err().to_string();
908        assert!(
909            err_msg.contains("external_account_authorized_user"),
910            "Error should mention unsupported credential type: {}",
911            err_msg
912        );
913    }
914
915    #[test]
916    fn gcs_test_with_credentials_skip_invalid_adc() {
917        use crate::StaticCredentialProvider;
918
919        // Create invalid ADC file with unsupported credential type
920        let mut invalid_adc_file = NamedTempFile::new().unwrap();
921        invalid_adc_file
922            .write_all(br#"{"type": "external_account_authorized_user", "audience": "test"}"#)
923            .unwrap();
924
925        // Create a custom credential provider
926        let custom_creds = Arc::new(StaticCredentialProvider::new(GcpCredential {
927            bearer: "custom-token".to_string(),
928        }));
929
930        // Build should succeed with custom credentials provider despite invalid ADC
931        let result = GoogleCloudStorageBuilder::new()
932            .with_credentials(custom_creds)
933            .with_application_credentials(invalid_adc_file.path().to_str().unwrap())
934            .with_bucket_name("test-bucket")
935            .build();
936
937        // Should succeed - ADC errors should be ignored when explicit creds provided via with_credentials
938        assert!(
939            result.is_ok(),
940            "Build should succeed with custom credentials despite invalid ADC: {:?}",
941            result.err()
942        );
943    }
944}
object_store/gcp/builder.rs

object_store/gcp/
builder.rs
