object_store/gcp/
builder.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::client::{
19    CryptoProvider, HttpConnector, TokenCredentialProvider, crypto_provider, http_connector,
20};
21use crate::config::ConfigValue;
22use crate::gcp::client::{GoogleCloudStorageClient, GoogleCloudStorageConfig};
23use crate::gcp::credential::{
24    ApplicationDefaultCredentials, DEFAULT_GCS_BASE_URL, InstanceCredentialProvider,
25    ServiceAccountCredentials,
26};
27use crate::gcp::{
28    GcpCredential, GcpCredentialProvider, GcpSigningCredential, GcpSigningCredentialProvider,
29    GoogleCloudStorage, STORE, credential,
30};
31use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
32use serde::{Deserialize, Serialize};
33use std::str::FromStr;
34use std::sync::Arc;
35use std::time::Duration;
36use url::Url;
37
38use super::credential::{AuthorizedUserSigningCredentials, InstanceSigningCredentialProvider};
39
40const TOKEN_MIN_TTL: Duration = Duration::from_secs(4 * 60);
41
42#[derive(Debug, thiserror::Error)]
43enum Error {
44    #[error("Missing bucket name")]
45    MissingBucketName {},
46
47    #[error("One of service account path or service account key may be provided.")]
48    ServiceAccountPathAndKeyProvided,
49
50    #[error("Unable parse source url. Url: {}, Error: {}", url, source)]
51    UnableToParseUrl {
52        source: url::ParseError,
53        url: String,
54    },
55
56    #[error(
57        "Unknown url scheme cannot be parsed into storage location: {}",
58        scheme
59    )]
60    UnknownUrlScheme { scheme: String },
61
62    #[error("URL did not match any known pattern for scheme: {}", url)]
63    UrlNotRecognised { url: String },
64
65    #[error("Configuration key: '{}' is not known.", key)]
66    UnknownConfigurationKey { key: String },
67
68    #[error("GCP credential error: {}", source)]
69    Credential { source: credential::Error },
70}
71
72impl From<Error> for crate::Error {
73    fn from(err: Error) -> Self {
74        match err {
75            Error::UnknownConfigurationKey { key } => {
76                Self::UnknownConfigurationKey { store: STORE, key }
77            }
78            _ => Self::Generic {
79                store: STORE,
80                source: Box::new(err),
81            },
82        }
83    }
84}
85
86/// Configure a connection to Google Cloud Storage.
87///
88/// If no credentials are explicitly provided, they will be sourced
89/// from the environment as documented [here](https://cloud.google.com/docs/authentication/application-default-credentials).
90///
91/// # Example
92/// ```
93/// # let BUCKET_NAME = "foo";
94/// # use object_store::gcp::GoogleCloudStorageBuilder;
95/// let gcs = GoogleCloudStorageBuilder::from_env().with_bucket_name(BUCKET_NAME).build();
96/// ```
97#[derive(Debug, Clone)]
98pub struct GoogleCloudStorageBuilder {
99    /// Bucket name
100    bucket_name: Option<String>,
101    /// Url
102    url: Option<String>,
103    /// Base URL
104    base_url: Option<String>,
105    /// Path to the service account file
106    service_account_path: Option<String>,
107    /// The serialized service account key
108    service_account_key: Option<String>,
109    /// Path to the application credentials file.
110    application_credentials_path: Option<String>,
111    /// Retry config
112    retry_config: RetryConfig,
113    /// Client options
114    client_options: ClientOptions,
115    /// Credentials
116    credentials: Option<GcpCredentialProvider>,
117    /// Explicit bearer token, if configured
118    bearer_token: Option<String>,
119    /// The [`CryptoProvider`] to use
120    crypto: Option<Arc<dyn CryptoProvider>>,
121    /// Skip signing requests
122    skip_signature: ConfigValue<bool>,
123    /// Credentials for sign url
124    signing_credentials: Option<GcpSigningCredentialProvider>,
125    /// The [`HttpConnector`] to use
126    http_connector: Option<Arc<dyn HttpConnector>>,
127}
128
129/// Configuration keys for [`GoogleCloudStorageBuilder`]
130///
131/// Configuration via keys can be done via [`GoogleCloudStorageBuilder::with_config`]
132///
133/// # Example
134/// ```
135/// # use object_store::gcp::{GoogleCloudStorageBuilder, GoogleConfigKey};
136/// let builder = GoogleCloudStorageBuilder::new()
137///     .with_config("google_service_account".parse().unwrap(), "my-service-account")
138///     .with_config(GoogleConfigKey::Bucket, "my-bucket");
139/// ```
140#[derive(PartialEq, Eq, Hash, Clone, Debug, Copy, Serialize, Deserialize)]
141#[non_exhaustive]
142pub enum GoogleConfigKey {
143    /// Path to the service account file
144    ///
145    /// Supported keys:
146    /// - `google_service_account`
147    /// - `service_account`
148    /// - `google_service_account_path`
149    /// - `service_account_path`
150    ServiceAccount,
151
152    /// The serialized service account key.
153    ///
154    /// Supported keys:
155    /// - `google_service_account_key`
156    /// - `service_account_key`
157    ServiceAccountKey,
158
159    /// Bucket name
160    ///
161    /// See [`GoogleCloudStorageBuilder::with_bucket_name`] for details.
162    ///
163    /// Supported keys:
164    /// - `google_bucket`
165    /// - `google_bucket_name`
166    /// - `bucket`
167    /// - `bucket_name`
168    Bucket,
169
170    /// Base URL
171    ///
172    /// See [`GoogleCloudStorageBuilder::with_base_url`] for details.
173    ///
174    /// Supported keys:
175    /// - `google_base_url`
176    /// - `base_url`
177    BaseUrl,
178
179    /// Application credentials path
180    ///
181    /// See [`GoogleCloudStorageBuilder::with_application_credentials`].
182    ///
183    /// Supported keys:
184    /// - `google_application_credentials`
185    /// - `application_credentials`
186    ApplicationCredentials,
187
188    /// Explicit OAuth bearer token
189    ///
190    /// This is treated as a static token and will not be refreshed automatically.
191    ///
192    /// Supported keys:
193    /// - `google_bearer_token`
194    /// - `bearer_token`
195    BearerToken,
196
197    /// Skip signing request
198    ///
199    /// Supported keys:
200    /// - `google_skip_signature`
201    /// - `skip_signature`
202    SkipSignature,
203
204    /// Client options
205    Client(ClientConfigKey),
206}
207
208impl AsRef<str> for GoogleConfigKey {
209    fn as_ref(&self) -> &str {
210        match self {
211            Self::ServiceAccount => "google_service_account",
212            Self::ServiceAccountKey => "google_service_account_key",
213            Self::Bucket => "google_bucket",
214            Self::BaseUrl => "google_base_url",
215            Self::ApplicationCredentials => "google_application_credentials",
216            Self::BearerToken => "google_bearer_token",
217            Self::SkipSignature => "google_skip_signature",
218            Self::Client(key) => key.as_ref(),
219        }
220    }
221}
222
223impl FromStr for GoogleConfigKey {
224    type Err = crate::Error;
225
226    fn from_str(s: &str) -> Result<Self, Self::Err> {
227        match s {
228            "google_service_account"
229            | "service_account"
230            | "google_service_account_path"
231            | "service_account_path" => Ok(Self::ServiceAccount),
232            "google_service_account_key" | "service_account_key" => Ok(Self::ServiceAccountKey),
233            "google_bucket" | "google_bucket_name" | "bucket" | "bucket_name" => Ok(Self::Bucket),
234            "google_base_url" | "base_url" => Ok(Self::BaseUrl),
235            "google_application_credentials" | "application_credentials" => {
236                Ok(Self::ApplicationCredentials)
237            }
238            "google_bearer_token" | "bearer_token" => Ok(Self::BearerToken),
239            "google_skip_signature" | "skip_signature" => Ok(Self::SkipSignature),
240            _ => match s.strip_prefix("google_").unwrap_or(s).parse() {
241                Ok(key) => Ok(Self::Client(key)),
242                Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
243            },
244        }
245    }
246}
247
248impl Default for GoogleCloudStorageBuilder {
249    fn default() -> Self {
250        Self {
251            bucket_name: None,
252            service_account_path: None,
253            service_account_key: None,
254            application_credentials_path: None,
255            retry_config: Default::default(),
256            client_options: ClientOptions::new().with_allow_http(true),
257            url: None,
258            base_url: None,
259            credentials: None,
260            bearer_token: None,
261            crypto: None,
262            skip_signature: Default::default(),
263            signing_credentials: None,
264            http_connector: None,
265        }
266    }
267}
268
269impl GoogleCloudStorageBuilder {
270    /// Create a new [`GoogleCloudStorageBuilder`] with default values.
271    pub fn new() -> Self {
272        Default::default()
273    }
274
275    /// Create an instance of [`GoogleCloudStorageBuilder`] with values pre-populated from environment variables.
276    ///
277    /// Variables extracted from environment:
278    /// * GOOGLE_SERVICE_ACCOUNT: location of service account file
279    /// * GOOGLE_SERVICE_ACCOUNT_PATH: (alias) location of service account file
280    /// * SERVICE_ACCOUNT: (alias) location of service account file
281    /// * GOOGLE_SERVICE_ACCOUNT_KEY: JSON serialized service account key
282    /// * GOOGLE_BUCKET: bucket name
283    /// * GOOGLE_BUCKET_NAME: (alias) bucket name
284    ///
285    /// # Example
286    /// ```
287    /// use object_store::gcp::GoogleCloudStorageBuilder;
288    ///
289    /// let gcs = GoogleCloudStorageBuilder::from_env()
290    ///     .with_bucket_name("foo")
291    ///     .build();
292    /// ```
293    pub fn from_env() -> Self {
294        let mut builder = Self::default();
295
296        if let Ok(service_account_path) = std::env::var("SERVICE_ACCOUNT") {
297            builder.service_account_path = Some(service_account_path);
298        }
299
300        for (os_key, os_value) in std::env::vars_os() {
301            if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
302                if key.starts_with("GOOGLE_") {
303                    if let Ok(config_key) = key.to_ascii_lowercase().parse() {
304                        builder = builder.with_config(config_key, value);
305                    }
306                }
307            }
308        }
309
310        builder
311    }
312
313    /// Parse available connection info form a well-known storage URL.
314    ///
315    /// The supported url schemes are:
316    ///
317    /// - `gs://<bucket>/<path>`
318    ///
319    /// Note: Settings derived from the URL will override any others set on this builder
320    ///
321    /// # Example
322    /// ```
323    /// use object_store::gcp::GoogleCloudStorageBuilder;
324    ///
325    /// let gcs = GoogleCloudStorageBuilder::from_env()
326    ///     .with_url("gs://bucket/path")
327    ///     .build();
328    /// ```
329    pub fn with_url(mut self, url: impl Into<String>) -> Self {
330        self.url = Some(url.into());
331        self
332    }
333
334    /// Set an option on the builder via a key - value pair.
335    pub fn with_config(mut self, key: GoogleConfigKey, value: impl Into<String>) -> Self {
336        match key {
337            GoogleConfigKey::ServiceAccount => self.service_account_path = Some(value.into()),
338            GoogleConfigKey::ServiceAccountKey => self.service_account_key = Some(value.into()),
339            GoogleConfigKey::Bucket => self.bucket_name = Some(value.into()),
340            GoogleConfigKey::BaseUrl => self.base_url = Some(value.into()),
341            GoogleConfigKey::ApplicationCredentials => {
342                self.application_credentials_path = Some(value.into())
343            }
344            GoogleConfigKey::BearerToken => self = self.with_bearer_token(value),
345            GoogleConfigKey::SkipSignature => self.skip_signature.parse(value),
346            GoogleConfigKey::Client(key) => {
347                self.client_options = self.client_options.with_config(key, value)
348            }
349        };
350        self
351    }
352
353    /// Get config value via a [`GoogleConfigKey`].
354    ///
355    /// # Example
356    /// ```
357    /// use object_store::gcp::{GoogleCloudStorageBuilder, GoogleConfigKey};
358    ///
359    /// let builder = GoogleCloudStorageBuilder::from_env()
360    ///     .with_service_account_key("foo");
361    /// let service_account_key = builder.get_config_value(&GoogleConfigKey::ServiceAccountKey).unwrap_or_default();
362    /// assert_eq!("foo", &service_account_key);
363    /// ```
364    pub fn get_config_value(&self, key: &GoogleConfigKey) -> Option<String> {
365        match key {
366            GoogleConfigKey::ServiceAccount => self.service_account_path.clone(),
367            GoogleConfigKey::ServiceAccountKey => self.service_account_key.clone(),
368            GoogleConfigKey::Bucket => self.bucket_name.clone(),
369            GoogleConfigKey::BaseUrl => self.base_url.clone(),
370            GoogleConfigKey::ApplicationCredentials => self.application_credentials_path.clone(),
371            GoogleConfigKey::BearerToken => self.bearer_token.clone(),
372            GoogleConfigKey::SkipSignature => Some(self.skip_signature.to_string()),
373            GoogleConfigKey::Client(key) => self.client_options.get_config_value(key),
374        }
375    }
376
377    /// Sets properties on this builder based on a URL
378    ///
379    /// This is a separate member function to allow fallible computation to
380    /// be deferred until [`Self::build`] which in turn allows deriving [`Clone`]
381    fn parse_url(&mut self, url: &str) -> Result<()> {
382        let parsed = Url::parse(url).map_err(|source| Error::UnableToParseUrl {
383            source,
384            url: url.to_string(),
385        })?;
386
387        let host = parsed.host_str().ok_or_else(|| Error::UrlNotRecognised {
388            url: url.to_string(),
389        })?;
390
391        match parsed.scheme() {
392            "gs" => self.bucket_name = Some(host.to_string()),
393            scheme => {
394                let scheme = scheme.to_string();
395                return Err(Error::UnknownUrlScheme { scheme }.into());
396            }
397        }
398        Ok(())
399    }
400
401    /// Set the bucket name (required)
402    pub fn with_bucket_name(mut self, bucket_name: impl Into<String>) -> Self {
403        self.bucket_name = Some(bucket_name.into());
404        self
405    }
406
407    /// Sets the base URL for communicating with GCS.
408    ///
409    /// If not explicitly set, it will be:
410    /// 1. Derived from the service account credentials, if provided
411    /// 2. Otherwise, uses the default GCS endpoint
412    ///
413    /// # Example
414    /// ```
415    /// use object_store::gcp::GoogleCloudStorageBuilder;
416    ///
417    /// let gcs = GoogleCloudStorageBuilder::from_env()
418    ///     .with_base_url("https://localhost:4443")
419    ///     .build();
420    /// ```
421    pub fn with_base_url(mut self, base_url: &str) -> Self {
422        self.base_url = Some(base_url.into());
423        self
424    }
425
426    /// Set the path to the service account file.
427    ///
428    /// This or [`GoogleCloudStorageBuilder::with_service_account_key`] must be
429    /// set.
430    ///
431    /// Example `"/tmp/gcs.json"`.
432    ///
433    /// Example contents of `gcs.json`:
434    ///
435    /// ```json
436    /// {
437    ///    "gcs_base_url": "https://localhost:4443",
438    ///    "disable_oauth": true,
439    ///    "client_email": "",
440    ///    "private_key": ""
441    /// }
442    /// ```
443    pub fn with_service_account_path(mut self, service_account_path: impl Into<String>) -> Self {
444        self.service_account_path = Some(service_account_path.into());
445        self
446    }
447
448    /// Set the service account key. The service account must be in the JSON
449    /// format.
450    ///
451    /// This or [`GoogleCloudStorageBuilder::with_service_account_path`] must be
452    /// set.
453    pub fn with_service_account_key(mut self, service_account: impl Into<String>) -> Self {
454        self.service_account_key = Some(service_account.into());
455        self
456    }
457
458    /// Set the path to the application credentials file.
459    ///
460    /// <https://cloud.google.com/docs/authentication/provide-credentials-adc>
461    pub fn with_application_credentials(
462        mut self,
463        application_credentials_path: impl Into<String>,
464    ) -> Self {
465        self.application_credentials_path = Some(application_credentials_path.into());
466        self
467    }
468
469    /// Set an explicit OAuth bearer token.
470    ///
471    /// This is treated as a static token and will not be refreshed automatically.
472    pub fn with_bearer_token(mut self, bearer_token: impl Into<String>) -> Self {
473        let bearer_token = bearer_token.into();
474        self.credentials = Some(Arc::new(StaticCredentialProvider::new(GcpCredential {
475            bearer: bearer_token.clone(),
476        })));
477        self.bearer_token = Some(bearer_token);
478        self
479    }
480
481    /// If enabled, [`GoogleCloudStorage`] will not fetch credentials and will not sign requests.
482    ///
483    /// This can be useful when interacting with public GCS buckets that deny authorized requests.
484    pub fn with_skip_signature(mut self, skip_signature: bool) -> Self {
485        self.skip_signature = skip_signature.into();
486        self
487    }
488
489    /// Set the credential provider overriding any other options
490    pub fn with_credentials(mut self, credentials: GcpCredentialProvider) -> Self {
491        self.credentials = Some(credentials);
492        self.bearer_token = None;
493        self
494    }
495
496    /// Set the signing credential provider overriding any other options
497    pub fn with_signing_credentials(mut self, credentials: GcpSigningCredentialProvider) -> Self {
498        self.signing_credentials = Some(credentials);
499        self
500    }
501
502    /// The [`CryptoProvider`] to use
503    pub fn with_crypto_provider(mut self, provider: Arc<dyn CryptoProvider>) -> Self {
504        self.crypto = Some(provider);
505        self
506    }
507
508    /// Set the retry configuration
509    pub fn with_retry(mut self, retry_config: RetryConfig) -> Self {
510        self.retry_config = retry_config;
511        self
512    }
513
514    /// Set the proxy_url to be used by the underlying client
515    pub fn with_proxy_url(mut self, proxy_url: impl Into<String>) -> Self {
516        self.client_options = self.client_options.with_proxy_url(proxy_url);
517        self
518    }
519
520    /// Set a trusted proxy CA certificate
521    pub fn with_proxy_ca_certificate(mut self, proxy_ca_certificate: impl Into<String>) -> Self {
522        self.client_options = self
523            .client_options
524            .with_proxy_ca_certificate(proxy_ca_certificate);
525        self
526    }
527
528    /// Set a list of hosts to exclude from proxy connections
529    pub fn with_proxy_excludes(mut self, proxy_excludes: impl Into<String>) -> Self {
530        self.client_options = self.client_options.with_proxy_excludes(proxy_excludes);
531        self
532    }
533
534    /// Sets the client options, overriding any already set
535    pub fn with_client_options(mut self, options: ClientOptions) -> Self {
536        self.client_options = options;
537        self
538    }
539
540    /// The [`HttpConnector`] to use
541    ///
542    /// On non-WASM32 platforms uses [`reqwest`] by default, on WASM32 platforms must be provided
543    pub fn with_http_connector<C: HttpConnector>(mut self, connector: C) -> Self {
544        self.http_connector = Some(Arc::new(connector));
545        self
546    }
547
548    /// Configure a connection to Google Cloud Storage, returning a
549    /// new [`GoogleCloudStorage`] and consuming `self`
550    pub fn build(mut self) -> Result<GoogleCloudStorage> {
551        if let Some(url) = self.url.take() {
552            self.parse_url(&url)?;
553        }
554
555        let bucket_name = self.bucket_name.ok_or(Error::MissingBucketName {})?;
556        let http = http_connector(self.http_connector)?;
557
558        // First try to initialize from the service account information.
559        let service_account_credentials =
560            match (self.service_account_path, self.service_account_key) {
561                (Some(path), None) => Some(
562                    ServiceAccountCredentials::from_file(path)
563                        .map_err(|source| Error::Credential { source })?,
564                ),
565                (None, Some(key)) => Some(
566                    ServiceAccountCredentials::from_key(&key)
567                        .map_err(|source| Error::Credential { source })?,
568                ),
569                (None, None) => None,
570                (Some(_), Some(_)) => return Err(Error::ServiceAccountPathAndKeyProvided.into()),
571            };
572
573        // Then try to initialize from the application credentials file, or the environment.
574        // Only attempt to read ADC if no explicit credentials were provided
575        let application_default_credentials =
576            if service_account_credentials.is_none() && self.credentials.is_none() {
577                // No explicit credentials, so try ADC and propagate errors
578                ApplicationDefaultCredentials::read(self.application_credentials_path.as_deref())?
579            } else {
580                // Explicit credentials provided, skip ADC reading entirely
581                None
582            };
583
584        let disable_oauth = service_account_credentials
585            .as_ref()
586            .map(|c| c.disable_oauth)
587            .unwrap_or(false);
588
589        let gcs_base_url: String = self
590            .base_url
591            .or_else(|| {
592                service_account_credentials
593                    .as_ref()
594                    .and_then(|c| c.gcs_base_url.clone())
595            })
596            .unwrap_or_else(|| DEFAULT_GCS_BASE_URL.to_string());
597
598        let credentials = if let Some(credentials) = self.credentials {
599            credentials
600        } else if disable_oauth {
601            Arc::new(StaticCredentialProvider::new(GcpCredential {
602                bearer: "".to_string(),
603            })) as _
604        } else if let Some(credentials) = service_account_credentials.clone() {
605            let crypto = crypto_provider(self.crypto.as_deref())?;
606            Arc::new(TokenCredentialProvider::new(
607                credentials.token_provider(crypto)?,
608                http.connect(&self.client_options)?,
609                self.retry_config.clone(),
610            )) as _
611        } else if let Some(credentials) = application_default_credentials.clone() {
612            match credentials {
613                ApplicationDefaultCredentials::AuthorizedUser(token) => Arc::new(
614                    TokenCredentialProvider::new(
615                        token,
616                        http.connect(&self.client_options)?,
617                        self.retry_config.clone(),
618                    )
619                    .with_min_ttl(TOKEN_MIN_TTL),
620                ) as _,
621                ApplicationDefaultCredentials::ServiceAccount(token) => {
622                    let crypto = crypto_provider(self.crypto.as_deref())?;
623                    Arc::new(TokenCredentialProvider::new(
624                        token.token_provider(crypto)?,
625                        http.connect(&self.client_options)?,
626                        self.retry_config.clone(),
627                    )) as _
628                }
629            }
630        } else {
631            Arc::new(
632                TokenCredentialProvider::new(
633                    InstanceCredentialProvider::default(),
634                    http.connect(&self.client_options.metadata_options())?,
635                    self.retry_config.clone(),
636                )
637                .with_min_ttl(TOKEN_MIN_TTL),
638            ) as _
639        };
640
641        let signing_credentials = if let Some(signing_credentials) = self.signing_credentials {
642            signing_credentials
643        } else if disable_oauth {
644            Arc::new(StaticCredentialProvider::new(GcpSigningCredential {
645                email: "".to_string(),
646                private_key: None,
647            })) as _
648        } else if let Some(credentials) = service_account_credentials.clone() {
649            let crypto = crypto_provider(self.crypto.as_deref())?;
650            credentials.signing_credentials(crypto)?
651        } else if let Some(credentials) = application_default_credentials.clone() {
652            match credentials {
653                ApplicationDefaultCredentials::AuthorizedUser(token) => {
654                    Arc::new(TokenCredentialProvider::new(
655                        AuthorizedUserSigningCredentials::from(token)?,
656                        http.connect(&self.client_options)?,
657                        self.retry_config.clone(),
658                    )) as _
659                }
660                ApplicationDefaultCredentials::ServiceAccount(token) => {
661                    let crypto = crypto_provider(self.crypto.as_deref())?;
662                    token.signing_credentials(crypto)?
663                }
664            }
665        } else {
666            Arc::new(TokenCredentialProvider::new(
667                InstanceSigningCredentialProvider::default(),
668                http.connect(&self.client_options.metadata_options())?,
669                self.retry_config.clone(),
670            )) as _
671        };
672
673        let config = GoogleCloudStorageConfig {
674            base_url: gcs_base_url,
675            credentials,
676            signing_credentials,
677            bucket_name,
678            crypto: self.crypto,
679            retry_config: self.retry_config,
680            client_options: self.client_options,
681            skip_signature: self.skip_signature.get()?,
682        };
683
684        let http_client = http.connect(&config.client_options)?;
685        Ok(GoogleCloudStorage {
686            client: Arc::new(GoogleCloudStorageClient::new(config, http_client)?),
687        })
688    }
689}
690
691#[cfg(test)]
692mod tests {
693    use super::*;
694    use std::collections::HashMap;
695    use std::io::Write;
696    use tempfile::NamedTempFile;
697
698    const FAKE_KEY: &str = r#"{"private_key": "private_key", "private_key_id": "private_key_id", "client_email":"client_email", "disable_oauth":true}"#;
699    const FAKE_KEY_WITH_BASE_URL: &str = r#"{"private_key": "private_key", "private_key_id": "private_key_id", "client_email":"client_email", "disable_oauth":true, "gcs_base_url": "https://base-url-from-credentials:4443"}"#;
700
701    #[test]
702    fn gcs_test_service_account_key_and_path() {
703        let mut tfile = NamedTempFile::new().unwrap();
704        write!(tfile, "{FAKE_KEY}").unwrap();
705        let _ = GoogleCloudStorageBuilder::new()
706            .with_service_account_key(FAKE_KEY)
707            .with_service_account_path(tfile.path().to_str().unwrap())
708            .with_bucket_name("foo")
709            .build()
710            .unwrap_err();
711    }
712
713    #[test]
714    fn gcs_test_config_from_map() {
715        let google_service_account = "object_store:fake_service_account".to_string();
716        let google_bucket_name = "object_store:fake_bucket".to_string();
717        let options = HashMap::from([
718            ("google_service_account", google_service_account.clone()),
719            ("google_bucket_name", google_bucket_name.clone()),
720        ]);
721
722        let builder = options
723            .iter()
724            .fold(GoogleCloudStorageBuilder::new(), |builder, (key, value)| {
725                builder.with_config(key.parse().unwrap(), value)
726            });
727
728        assert_eq!(
729            builder.service_account_path.unwrap(),
730            google_service_account.as_str()
731        );
732        assert_eq!(builder.bucket_name.unwrap(), google_bucket_name.as_str());
733    }
734
735    #[tokio::test]
736    async fn gcs_test_config_aliases() {
737        // Service account path
738        for alias in [
739            "google_service_account",
740            "service_account",
741            "google_service_account_path",
742            "service_account_path",
743        ] {
744            let builder = GoogleCloudStorageBuilder::new()
745                .with_config(alias.parse().unwrap(), "/fake/path.json");
746            assert_eq!("/fake/path.json", builder.service_account_path.unwrap());
747        }
748
749        // Service account key
750        for alias in ["google_service_account_key", "service_account_key"] {
751            let builder =
752                GoogleCloudStorageBuilder::new().with_config(alias.parse().unwrap(), FAKE_KEY);
753            assert_eq!(FAKE_KEY, builder.service_account_key.unwrap());
754        }
755
756        // Bucket name
757        for alias in [
758            "google_bucket",
759            "google_bucket_name",
760            "bucket",
761            "bucket_name",
762        ] {
763            let builder =
764                GoogleCloudStorageBuilder::new().with_config(alias.parse().unwrap(), "fake_bucket");
765            assert_eq!("fake_bucket", builder.bucket_name.unwrap());
766        }
767
768        for alias in ["google_bearer_token", "bearer_token"] {
769            let gcs = GoogleCloudStorageBuilder::new()
770                .with_config(alias.parse().unwrap(), "test-token")
771                .with_bucket_name("fake_bucket")
772                .build()
773                .unwrap();
774
775            let credential = gcs.credentials().get_credential().await.unwrap();
776            assert_eq!(credential.bearer, "test-token");
777        }
778    }
779
780    #[cfg(feature = "reqwest")]
781    #[tokio::test]
782    async fn gcs_test_proxy_url() {
783        let mut tfile = NamedTempFile::new().unwrap();
784        write!(tfile, "{FAKE_KEY}").unwrap();
785        let service_account_path = tfile.path();
786        let gcs = GoogleCloudStorageBuilder::new()
787            .with_service_account_path(service_account_path.to_str().unwrap())
788            .with_bucket_name("foo")
789            .with_proxy_url("https://example.com")
790            .build();
791        assert!(gcs.is_ok());
792
793        let err = GoogleCloudStorageBuilder::new()
794            .with_service_account_path(service_account_path.to_str().unwrap())
795            .with_bucket_name("foo")
796            // use invalid url
797            .with_proxy_url("dxx:ddd\\example.com")
798            .build()
799            .unwrap_err()
800            .to_string();
801
802        assert_eq!("Generic HTTP client error: builder error", err);
803    }
804
805    #[test]
806    fn gcs_test_urls() {
807        let mut builder = GoogleCloudStorageBuilder::new();
808        builder.parse_url("gs://bucket/path").unwrap();
809        assert_eq!(builder.bucket_name.as_deref(), Some("bucket"));
810
811        builder.parse_url("gs://bucket.mydomain/path").unwrap();
812        assert_eq!(builder.bucket_name.as_deref(), Some("bucket.mydomain"));
813
814        builder.parse_url("mailto://bucket/path").unwrap_err();
815    }
816
817    #[cfg(feature = "reqwest")]
818    #[test]
819    fn gcs_test_service_account_key_only() {
820        let _ = GoogleCloudStorageBuilder::new()
821            .with_service_account_key(FAKE_KEY)
822            .with_bucket_name("foo")
823            .build()
824            .unwrap();
825    }
826
827    #[cfg(feature = "reqwest")]
828    #[test]
829    fn gcs_test_with_base_url() {
830        let no_base_url = GoogleCloudStorageBuilder::new()
831            .with_bucket_name("foo")
832            .build()
833            .unwrap();
834        assert_eq!(no_base_url.client.config().base_url, DEFAULT_GCS_BASE_URL);
835
836        let explicit_override = GoogleCloudStorageBuilder::new()
837            .with_bucket_name("foo")
838            .with_base_url("https://explicitly-overridden:4443")
839            .build()
840            .unwrap();
841        assert_eq!(
842            explicit_override.client.config().base_url,
843            "https://explicitly-overridden:4443"
844        );
845
846        let url_in_credentials = GoogleCloudStorageBuilder::new()
847            .with_bucket_name("foo")
848            .with_service_account_key(FAKE_KEY_WITH_BASE_URL)
849            .build()
850            .unwrap();
851        assert_eq!(
852            url_in_credentials.client.config().base_url,
853            "https://base-url-from-credentials:4443"
854        );
855
856        let explicit_override_and_credentials = GoogleCloudStorageBuilder::new()
857            .with_bucket_name("foo")
858            .with_base_url("https://explicitly-overridden:4443") // this should take precedence
859            .with_service_account_key(FAKE_KEY_WITH_BASE_URL)
860            .build()
861            .unwrap();
862        assert_eq!(
863            explicit_override_and_credentials.client.config().base_url,
864            "https://explicitly-overridden:4443"
865        );
866    }
867
868    #[test]
869    fn gcs_test_config_get_value() {
870        let google_service_account = "object_store:fake_service_account".to_string();
871        let google_bucket_name = "object_store:fake_bucket".to_string();
872        let google_bearer_token = "test-token".to_string();
873        let builder = GoogleCloudStorageBuilder::new()
874            .with_config(GoogleConfigKey::ServiceAccount, &google_service_account)
875            .with_config(GoogleConfigKey::Bucket, &google_bucket_name)
876            .with_config(GoogleConfigKey::BearerToken, &google_bearer_token);
877
878        assert_eq!(
879            builder
880                .get_config_value(&GoogleConfigKey::ServiceAccount)
881                .unwrap(),
882            google_service_account
883        );
884        assert_eq!(
885            builder.get_config_value(&GoogleConfigKey::Bucket).unwrap(),
886            google_bucket_name
887        );
888        assert_eq!(
889            builder
890                .get_config_value(&GoogleConfigKey::BearerToken)
891                .unwrap(),
892            google_bearer_token
893        );
894    }
895
896    #[test]
897    fn gcs_test_with_credentials_clears_bearer_config_value() {
898        let custom_creds = Arc::new(StaticCredentialProvider::new(GcpCredential {
899            bearer: "custom-token".to_string(),
900        }));
901
902        let builder = GoogleCloudStorageBuilder::new()
903            .with_bearer_token("test-token")
904            .with_credentials(custom_creds);
905
906        assert_eq!(
907            builder.get_config_value(&GoogleConfigKey::BearerToken),
908            None
909        );
910    }
911
912    #[test]
913    fn gcp_test_client_opts() {
914        let key = "GOOGLE_PROXY_URL";
915        if let Ok(config_key) = key.to_ascii_lowercase().parse() {
916            assert_eq!(
917                GoogleConfigKey::Client(ClientConfigKey::ProxyUrl),
918                config_key
919            );
920        } else {
921            panic!("{key} not propagated as ClientConfigKey");
922        }
923    }
924
925    #[cfg(feature = "reqwest")]
926    #[test]
927    fn gcs_test_explicit_creds_skip_invalid_adc() {
928        // Create a valid service account key file
929        let mut valid_key_file = NamedTempFile::new().unwrap();
930        write!(valid_key_file, "{FAKE_KEY}").unwrap();
931
932        // Create invalid ADC file with unsupported credential type
933        let mut invalid_adc_file = NamedTempFile::new().unwrap();
934        invalid_adc_file
935            .write_all(br#"{"type": "external_account_authorized_user", "audience": "test"}"#)
936            .unwrap();
937
938        // Build should succeed because explicit credentials are provided
939        // and ADC errors should be ignored
940        let result = GoogleCloudStorageBuilder::new()
941            .with_service_account_path(valid_key_file.path().to_str().unwrap())
942            .with_application_credentials(invalid_adc_file.path().to_str().unwrap())
943            .with_bucket_name("test-bucket")
944            .build();
945
946        // Should succeed - ADC errors should be ignored when explicit creds provided
947        assert!(
948            result.is_ok(),
949            "Build should succeed with explicit credentials despite invalid ADC: {:?}",
950            result.err()
951        );
952    }
953
954    #[cfg(feature = "reqwest")]
955    #[test]
956    fn gcs_test_explicit_creds_with_service_account_key_skip_invalid_adc() {
957        // Create invalid ADC file with unsupported credential type
958        let mut invalid_adc_file = NamedTempFile::new().unwrap();
959        invalid_adc_file
960            .write_all(br#"{"type": "external_account_authorized_user", "audience": "test"}"#)
961            .unwrap();
962
963        // Build should succeed with service account key (not path)
964        let result = GoogleCloudStorageBuilder::new()
965            .with_service_account_key(FAKE_KEY)
966            .with_application_credentials(invalid_adc_file.path().to_str().unwrap())
967            .with_bucket_name("test-bucket")
968            .build();
969
970        // Should succeed - ADC errors should be ignored when explicit creds provided
971        assert!(
972            result.is_ok(),
973            "Build should succeed with service account key despite invalid ADC: {:?}",
974            result.err()
975        );
976    }
977
978    #[cfg(feature = "reqwest")]
979    #[test]
980    fn gcs_test_adc_error_propagated_without_explicit_creds() {
981        // Create invalid ADC file with unsupported credential type
982        let mut invalid_adc_file = NamedTempFile::new().unwrap();
983        invalid_adc_file
984            .write_all(br#"{"type": "external_account_authorized_user", "audience": "test"}"#)
985            .unwrap();
986
987        // Build should fail because no explicit credentials and ADC is invalid
988        let result = GoogleCloudStorageBuilder::new()
989            .with_application_credentials(invalid_adc_file.path().to_str().unwrap())
990            .with_bucket_name("test-bucket")
991            .build();
992
993        // Should fail - ADC errors should be propagated when no explicit creds
994        assert!(
995            result.is_err(),
996            "Build should fail without explicit credentials and invalid ADC"
997        );
998        let err_msg = result.unwrap_err().to_string();
999        assert!(
1000            err_msg.contains("external_account_authorized_user"),
1001            "Error should mention unsupported credential type: {}",
1002            err_msg
1003        );
1004    }
1005
1006    #[cfg(feature = "reqwest")]
1007    #[test]
1008    fn gcs_test_with_credentials_skip_invalid_adc() {
1009        use crate::StaticCredentialProvider;
1010
1011        // Create invalid ADC file with unsupported credential type
1012        let mut invalid_adc_file = NamedTempFile::new().unwrap();
1013        invalid_adc_file
1014            .write_all(br#"{"type": "external_account_authorized_user", "audience": "test"}"#)
1015            .unwrap();
1016
1017        // Create a custom credential provider
1018        let custom_creds = Arc::new(StaticCredentialProvider::new(GcpCredential {
1019            bearer: "custom-token".to_string(),
1020        }));
1021
1022        // Build should succeed with custom credentials provider despite invalid ADC
1023        let result = GoogleCloudStorageBuilder::new()
1024            .with_credentials(custom_creds)
1025            .with_application_credentials(invalid_adc_file.path().to_str().unwrap())
1026            .with_bucket_name("test-bucket")
1027            .build();
1028
1029        // Should succeed - ADC errors should be ignored when explicit creds provided via with_credentials
1030        assert!(
1031            result.is_ok(),
1032            "Build should succeed with custom credentials despite invalid ADC: {:?}",
1033            result.err()
1034        );
1035    }
1036}
object_store/gcp/builder.rs

object_store/gcp/
builder.rs
