gcp_bigquery_client/
lib.rs

1//! [<img alt="github" src="https://img.shields.io/badge/github-lquerel/gcp_bigquery_client-8da0cb?style=for-the-badge&labelColor=555555&logo=github" height="20">](https://github.com/lquerel/gcp-bigquery-client)
2//! [<img alt="crates.io" src="https://img.shields.io/crates/v/gcp_bigquery_client.svg?style=for-the-badge&color=fc8d62&logo=rust" height="20">](https://crates.io/crates/gcp-bigquery-client)
3//! [<img alt="build status" src="https://img.shields.io/github/workflow/status/lquerel/gcp-bigquery-client/Rust/main?style=for-the-badge" height="20">](https://github.com/lquerel/gcp-bigquery-client/actions?query=branch%3Amain)
4//!
5//! An ergonomic async client library for GCP BigQuery.
6//! * Support for dataset, table, streaming API and query (see [status section](#status) for an exhaustive list of supported API endpoints)
7//! * Support Service Account Key authentication (other OAuth flows will be added later)
8//! * Create tables and rows via builder patterns
9//! * Persist complex Rust structs in structured BigQuery tables
10//! * Async API
11//!
12//! <br>
13//!
14//! Other OAuth flows will be added later.
15//!
16//! For a detailed tutorial on the different ways to use GCP BigQuery Client please check out the [GitHub repository](https://github.com/lquerel/gcp-bigquery-client).
17
18#![allow(clippy::result_large_err)]
19
20#[macro_use]
21extern crate serde;
22extern crate serde_json;
23
24use std::env;
25use std::path::PathBuf;
26use std::sync::Arc;
27
28use client_builder::ClientBuilder;
29use reqwest::Response;
30use serde::Deserialize;
31use storage::StorageApi;
32use yup_oauth2::ServiceAccountKey;
33
34use crate::auth::Authenticator;
35use crate::dataset::DatasetApi;
36use crate::error::BQError;
37use crate::job::JobApi;
38use crate::model_api::ModelApi;
39use crate::project::ProjectApi;
40use crate::routine::RoutineApi;
41use crate::table::TableApi;
42use crate::tabledata::TableDataApi;
43
44/// Since yup_oauth2 structs are used as parameters in public functions there is already semver
45/// coupling, as it is an error if consumer uses different version of yup_oauth than gcp-bigquery-client
46/// Export yup_oauth2 so consumers don't need to carefully keep their dependency versions in sync.
47/// (see https://github.com/lquerel/gcp-bigquery-client/pull/86)
48pub use yup_oauth2;
49
50pub mod auth;
51pub mod client_builder;
52pub mod dataset;
53pub mod error;
54pub mod job;
55pub mod model;
56pub mod model_api;
57pub mod project;
58pub mod routine;
59pub mod storage;
60pub mod table;
61pub mod tabledata;
62
63const BIG_QUERY_V2_URL: &str = "https://bigquery.googleapis.com/bigquery/v2";
64const BIG_QUERY_AUTH_URL: &str = "https://www.googleapis.com/auth/bigquery";
65
66/// An asynchronous BigQuery client.
67#[derive(Clone)]
68pub struct Client {
69    dataset_api: DatasetApi,
70    table_api: TableApi,
71    job_api: JobApi,
72    tabledata_api: TableDataApi,
73    routine_api: RoutineApi,
74    model_api: ModelApi,
75    project_api: ProjectApi,
76    storage_api: StorageApi,
77}
78
79impl Client {
80    pub async fn from_authenticator(auth: Arc<dyn Authenticator>) -> Result<Self, BQError> {
81        let client = reqwest::Client::new();
82
83        Ok(Self {
84            dataset_api: DatasetApi::new(client.clone(), Arc::clone(&auth)),
85            table_api: TableApi::new(client.clone(), Arc::clone(&auth)),
86            job_api: JobApi::new(client.clone(), Arc::clone(&auth)),
87            tabledata_api: TableDataApi::new(client.clone(), Arc::clone(&auth)),
88            routine_api: RoutineApi::new(client.clone(), Arc::clone(&auth)),
89            model_api: ModelApi::new(client.clone(), Arc::clone(&auth)),
90            project_api: ProjectApi::new(client, Arc::clone(&auth)),
91            storage_api: StorageApi::new(auth).await?,
92        })
93    }
94
95    /// Constructs a new BigQuery client.
96    /// # Argument
97    /// * `sa_key_file` - A GCP Service Account Key file.
98    pub async fn from_service_account_key_file(sa_key_file: &str) -> Result<Self, BQError> {
99        ClientBuilder::new()
100            .build_from_service_account_key_file(sa_key_file)
101            .await
102    }
103
104    /// Constructs a new BigQuery client from a [`ServiceAccountKey`].
105    /// # Argument
106    /// * `sa_key` - A GCP Service Account Key `yup-oauth2` object.
107    /// * `readonly` - A boolean setting whether the acquired token scope should be readonly.
108    ///
109    /// [`ServiceAccountKey`]: https://docs.rs/yup-oauth2/*/yup_oauth2/struct.ServiceAccountKey.html
110    pub async fn from_service_account_key(sa_key: ServiceAccountKey, readonly: bool) -> Result<Self, BQError> {
111        ClientBuilder::new()
112            .build_from_service_account_key(sa_key, readonly)
113            .await
114    }
115
116    pub async fn with_workload_identity(readonly: bool) -> Result<Self, BQError> {
117        ClientBuilder::new().build_with_workload_identity(readonly).await
118    }
119
120    pub(crate) fn v2_base_url(&mut self, base_url: String) -> &mut Self {
121        self.dataset_api.with_base_url(base_url.clone());
122        self.table_api.with_base_url(base_url.clone());
123        self.job_api.with_base_url(base_url.clone());
124        self.tabledata_api.with_base_url(base_url.clone());
125        self.routine_api.with_base_url(base_url.clone());
126        self.model_api.with_base_url(base_url.clone());
127        self.project_api.with_base_url(base_url.clone());
128        self.storage_api.with_base_url(base_url);
129        self
130    }
131
132    pub async fn from_installed_flow_authenticator<S: AsRef<[u8]>, P: Into<PathBuf>>(
133        secret: S,
134        persistant_file_path: P,
135    ) -> Result<Self, BQError> {
136        ClientBuilder::new()
137            .build_from_installed_flow_authenticator(secret, persistant_file_path)
138            .await
139    }
140
141    pub async fn from_installed_flow_authenticator_from_secret_file<P: Into<PathBuf>>(
142        secret_file: &str,
143        persistant_file_path: P,
144    ) -> Result<Self, BQError> {
145        Self::from_installed_flow_authenticator(
146            tokio::fs::read(secret_file)
147                .await
148                .expect("expecting a valid secret file."),
149            persistant_file_path,
150        )
151        .await
152    }
153
154    pub async fn from_application_default_credentials() -> Result<Self, BQError> {
155        ClientBuilder::new().build_from_application_default_credentials().await
156    }
157
158    pub async fn from_authorized_user_secret(secret: &str) -> Result<Self, BQError> {
159        ClientBuilder::new()
160            .build_from_authorized_user_authenticator(secret)
161            .await
162    }
163
164    /// Returns a dataset API handler.
165    pub fn dataset(&self) -> &DatasetApi {
166        &self.dataset_api
167    }
168
169    /// Returns a table API handler.
170    pub fn table(&self) -> &TableApi {
171        &self.table_api
172    }
173
174    /// Returns a job API handler.
175    pub fn job(&self) -> &JobApi {
176        &self.job_api
177    }
178
179    /// Returns a table data API handler.
180    pub fn tabledata(&self) -> &TableDataApi {
181        &self.tabledata_api
182    }
183
184    /// Returns a routine API handler.
185    pub fn routine(&self) -> &RoutineApi {
186        &self.routine_api
187    }
188
189    /// Returns a model API handler.
190    pub fn model(&self) -> &ModelApi {
191        &self.model_api
192    }
193
194    /// Returns a project API handler.
195    pub fn project(&self) -> &ProjectApi {
196        &self.project_api
197    }
198
199    /// Returns a storage API handler.
200    pub fn storage(&self) -> &StorageApi {
201        &self.storage_api
202    }
203
204    /// Returns a mutable storage API handler.
205    pub fn storage_mut(&mut self) -> &mut StorageApi {
206        &mut self.storage_api
207    }
208}
209
210pub(crate) fn urlencode<T: AsRef<str>>(s: T) -> String {
211    url::form_urlencoded::byte_serialize(s.as_ref().as_bytes()).collect()
212}
213
214async fn process_response<T: for<'de> Deserialize<'de>>(resp: Response) -> Result<T, BQError> {
215    if resp.status().is_success() {
216        Ok(resp.json().await?)
217    } else {
218        Err(BQError::ResponseError {
219            error: resp.json().await?,
220        })
221    }
222}
223
224pub fn env_vars() -> (String, String, String, String) {
225    let project_id = env::var("PROJECT_ID").expect("Environment variable PROJECT_ID");
226    let dataset_id = env::var("DATASET_ID").expect("Environment variable DATASET_ID");
227    let table_id = env::var("TABLE_ID").expect("Environment variable TABLE_ID");
228    let gcp_sa_key =
229        env::var("GOOGLE_APPLICATION_CREDENTIALS").expect("Environment variable GOOGLE_APPLICATION_CREDENTIALS");
230
231    (project_id, dataset_id, table_id, gcp_sa_key)
232}
233
234pub mod google {
235    #![allow(clippy::all)]
236    #[path = "google.api.rs"]
237    pub mod api;
238
239    #[path = ""]
240    pub mod cloud {
241        #[path = ""]
242        pub mod bigquery {
243            #[path = ""]
244            pub mod storage {
245                #![allow(clippy::all)]
246                #[path = "google.cloud.bigquery.storage.v1.rs"]
247                pub mod v1;
248            }
249        }
250    }
251
252    #[path = "google.rpc.rs"]
253    pub mod rpc;
254}
gcp_bigquery_client/lib.rs

gcp_bigquery_client/
lib.rs
