datafusion_objectstore_s3/
lib.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18#![warn(missing_docs)]
19
20//! [DataFusion-ObjectStore-S3](https://github.com/datafusion-contrib/datafusion-objectstore-s3)
21//! provides a `TableProvider` interface for using `Datafusion` to query data in S3.  This includes AWS S3
22//! and services such as MinIO that implement the S3 API.
23//!
24//! ## Examples
25//! Examples for querying AWS and other implementors, such as MinIO, are shown below.
26//!
27//! Load credentials from default AWS credential provider (such as environment or ~/.aws/credentials)
28//!
29//! ```rust
30//! # use std::sync::Arc;
31//! # use datafusion::error::Result;
32//! # use datafusion_objectstore_s3::object_store::s3::S3FileSystem;
33//! # #[tokio::main]
34//! # async fn main() -> Result<()> {
35//! let s3_file_system = Arc::new(S3FileSystem::default().await);
36//! # Ok(())
37//! # }
38//! ```
39//!
40//! `S3FileSystem::default()` is a convenience wrapper for `S3FileSystem::new(None, None, None, None, None, None)`.
41//!
42//! Connect to implementor of S3 API (MinIO, in this case) using access key and secret.
43//!
44//! ```rust
45//! use datafusion_objectstore_s3::object_store::s3::S3FileSystem;
46//!
47//! use aws_types::credentials::SharedCredentialsProvider;
48//! use aws_types::credentials::Credentials;
49//! use aws_sdk_s3::Endpoint;
50//! use http::Uri;
51//!
52//! # #[tokio::main]
53//! # async fn main() {
54//! // Example credentials provided by MinIO
55//! const MINIO_ACCESS_KEY_ID: &str = "AKIAIOSFODNN7EXAMPLE";
56//! const MINIO_SECRET_ACCESS_KEY: &str = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
57//! const PROVIDER_NAME: &str = "Static";
58//! const MINIO_ENDPOINT: &str = "http://localhost:9000";
59//!
60//! let s3_file_system = S3FileSystem::new(
61//!     Some(SharedCredentialsProvider::new(Credentials::new(
62//!         MINIO_ACCESS_KEY_ID,
63//!         MINIO_SECRET_ACCESS_KEY,
64//!         None,
65//!         None,
66//!         PROVIDER_NAME,
67//!     ))), // SharedCredentialsProvider
68//!     None, //Region
69//!     Some(Endpoint::immutable(Uri::from_static(MINIO_ENDPOINT))), //Endpoint
70//!     None, // RetryConfig
71//!     None, // AsyncSleep
72//!     None, // TimeoutConfig
73//! )
74//! .await;
75//! # }
76//! ```
77//!
78//! Using DataFusion's `ListingOtions` and `ListingTable` we register a table into a DataFusion `ExecutionContext` so that it can be queried.
79//!
80//! ```rust
81//! use std::sync::Arc;
82//!
83//! use datafusion::datasource::listing::*;
84//! use datafusion::datasource::TableProvider;
85//! use datafusion::prelude::SessionContext;
86//! use datafusion::datasource::file_format::parquet::ParquetFormat;
87//! use datafusion::error::Result;
88//!
89//! use datafusion_objectstore_s3::object_store::s3::S3FileSystem;
90//!
91//! use aws_types::credentials::SharedCredentialsProvider;
92//! use aws_types::credentials::Credentials;
93//! use aws_sdk_s3::Endpoint;
94//! use http::Uri;
95//!
96//! # const MINIO_ACCESS_KEY_ID: &str = "AKIAIOSFODNN7EXAMPLE";
97//! # const MINIO_SECRET_ACCESS_KEY: &str = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
98//! # const PROVIDER_NAME: &str = "Static";
99//! # const MINIO_ENDPOINT: &str = "http://localhost:9000";
100//!
101//! # #[tokio::main]
102//! # async fn main() -> Result<()> {
103//! let filename = "s3://data/alltypes_plain.snappy.parquet";
104//!
105//! # let s3_file_system = Arc::new(S3FileSystem::new(
106//! #     Some(SharedCredentialsProvider::new(Credentials::new(
107//! #         MINIO_ACCESS_KEY_ID,
108//! #         MINIO_SECRET_ACCESS_KEY,
109//! #         None,
110//! #         None,
111//! #         PROVIDER_NAME,
112//! #     ))),
113//! #     None,
114//! #     Some(Endpoint::immutable(Uri::from_static(MINIO_ENDPOINT))),
115//! #     None,
116//! #     None,
117//! #     None,
118//! # )
119//! # .await);
120//!
121//! let config = ListingTableConfig::new(s3_file_system, filename).infer().await?;
122//!
123//! let table = ListingTable::try_new(config)?;
124//!
125//! let mut ctx = SessionContext::new();
126//!
127//! ctx.register_table("tbl", Arc::new(table))?;
128//!
129//! let df = ctx.sql("SELECT * FROM tbl").await?;
130//! df.show();
131//! # Ok(())
132//! # }
133//! ```
134//!
135//! We can also register the `S3FileSystem` directly as an `ObjectStore` on an `ExecutionContext`. This provides an idiomatic way of creating `TableProviders` that can be queried.
136//!
137//! ```rust
138//! use std::sync::Arc;
139//!
140//! use datafusion::datasource::listing::*;
141//! use datafusion::error::Result;
142//!
143//! use datafusion_objectstore_s3::object_store::s3::S3FileSystem;
144//!
145//! use aws_sdk_s3::Endpoint;
146//! use aws_types::credentials::Credentials;
147//! use aws_types::credentials::SharedCredentialsProvider;
148//! use datafusion::prelude::SessionContext;
149//! use http::Uri;
150//!
151//! const MINIO_ACCESS_KEY_ID: &str = "AKIAIOSFODNN7EXAMPLE";
152//! const MINIO_SECRET_ACCESS_KEY: &str = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
153//! const PROVIDER_NAME: &str = "Static";
154//! const MINIO_ENDPOINT: &str = "http://localhost:9000";
155//!
156//! #[tokio::main]
157//! async fn main() -> Result<()> {
158//!     let s3_file_system = Arc::new(
159//!         S3FileSystem::new(
160//!             Some(SharedCredentialsProvider::new(Credentials::new(
161//!                 MINIO_ACCESS_KEY_ID,
162//!                 MINIO_SECRET_ACCESS_KEY,
163//!                 None,
164//!                 None,
165//!                 PROVIDER_NAME,
166//!             ))),
167//!             None,
168//!             Some(Endpoint::immutable(Uri::from_static(MINIO_ENDPOINT))),
169//!             None,
170//!             None,
171//!             None,
172//!         )
173//!         .await,
174//!     );
175//!
176//!     let ctx = SessionContext::new();
177//!
178//!     let uri = "s3://data/alltypes_plain.snappy.parquet";
179//!
180//!     let config = ListingTableConfig::new(s3_file_system, uri)
181//!         .infer()
182//!         .await?;
183//!
184//!     let table = ListingTable::try_new(config)?;
185//!
186//!     ctx.register_table("tbl", Arc::new(table))?;
187//!
188//!     let df = ctx.sql("SELECT * FROM tbl").await?;
189//!     df.show().await?;
190//!     Ok(())
191//! }
192//! ```
193
194pub mod error;
195pub mod object_store;
datafusion_objectstore_s3/lib.rs

datafusion_objectstore_s3/
lib.rs
