1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
// Copyright 2022 Datafuse Labs.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! OpenDAL is the Open Data Access Layer to **freely**, **painlessly**, and **efficiently** access data.
//!
//! # Services
//!
//! `Service` represents a backend scheme that OpenDAL supported.
//!
//! OpenDAL supports the following services now:
//!
//! | Services | Description |
//! | -------- | ----------- |
//! | [azblob][services::azblob] | Azure Storage Blob services. |
//! | [fs][services::fs] | POSIX alike file system. |
//! | [ftp][services::ftp] | FTP and FTPS support. |
//! | [gcs][services::gcs] | Google Cloud Storage service. |
//! | [hdfs][services::hdfs] | Hadoop Distributed File System(HDFS). |
//! | [http][services::http] | HTTP read-only backend. |
//! | [ipfs][services::ipfs] | IPFS HTTP Gateway support. |
//! | [ipmfs][services::ipmfs] | IPFS Mutable File System support. |
//! | [memory][services::memory] | In memory backend support. |
//! | [moka][services::moka] | [moka](https://github.com/moka-rs/moka) backend support. |
//! | [obs][services::obs] | Huawei Cloud OBS service. |
//! | [oss][services::oss] | Aliyun Object Storage Service (OSS).|
//! | [redis][services::redis] | Redis service. |
//! | [rocksdb][services::rocksdb] | RocksDB service. |
//! | [s3][services::s3] | AWS S3 alike services. |
//!
//! More services support is tracked at [opendal#5](https://github.com/datafuselabs/opendal/issues/5)
//!
//! # Layers
//!
//! `Layer` is the mechanism to intercept operations.
//!
//! OpenDAL supports the following layers now:
//!
//! | Layers | Description |
//! | -------- | ----------- |
//! | [ConcurrentLimitLayer][layers::ConcurrentLimitLayer] | Concurrent request limit. |
//! | [ContentCacheLayer][layers::ContentCacheLayer] | Content cache. |
//! | [ImmutableIndexLayer][layers::ImmutableIndexLayer] | Immutable in-memory index. |
//! | [LoggingLayer][layers::LoggingLayer] | Logging for every operations. |
//! | [MetadataCacheLayer][layers::MetadataCacheLayer] | Metadata cache. |
//! | [MetricsLayer][layers::MetricsLayer] | Metrics for every operations. |
//! | [RetryLayer][layers::RetryLayer] | Retry for failed operations. |
//! | [SubdirLayer][layers::SubdirLayer] | Allow switching directory. |
//! | [TracingLayer][layers::TracingLayer] | Tracing for every operations. |
//!
//! # Optional features
//!
//! ## Layers
//!
//! - `layers-all`: Enable all layers support.
//! - `layers-metrics`: Enable operator metrics support.
//! - `layers-tracing`: Enable operator tracing support.
//!
//! ## Services
//!
//! - `services-ftp`: Enable ftp service support.
//! - `services-hdfs`: Enable hdfs service support.
//! - `services-moka`: Enable moka service support.
//! - `services-ipfs`: Enable ipfs service support.
//! - `services-redis`: Enable redis service support.
//! - `services-rocksdb`: Enable rocksdb service support.
//!
//! ## Dependencies features
//!
//! - `compress`: Enable object decompress read support.
//! - `rustls`: Enable TLS functionality provided by `rustls`, enabled by default
//! - `native-tls`: Enable TLS functionality provided by `native-tls`
//! - `native-tls-vendored`: Enable the `vendored` feature of `native-tls`
//!
//! # Examples
//!
//! More examples could be found at <https://opendal.databend.rs/examples/index.html>
//!
//! ```no_run
//! use anyhow::Result;
//! use backon::ExponentialBackoff;
//! use futures::StreamExt;
//! use futures::TryStreamExt;
//! use opendal::layers::LoggingLayer;
//! use opendal::layers::RetryLayer;
//! use opendal::Object;
//! use opendal::ObjectEntry;
//! use opendal::ObjectMetadata;
//! use opendal::ObjectMode;
//! use opendal::ObjectStreamer;
//! use opendal::Operator;
//! use opendal::Scheme;
//!
//! #[tokio::main]
//! async fn main() -> Result<()> {
//!     // Init a fs operator
//!     let op = Operator::from_env(Scheme::Fs)?
//!         // Init with logging layer enabled.
//!         .layer(LoggingLayer)
//!         // Init with retry layer enabled.
//!         .layer(RetryLayer::new(ExponentialBackoff::default()));
//!
//!     // Create object handler.
//!     let o = op.object("test_file");
//!
//!     // Write data
//!     o.write("Hello, World!").await?;
//!
//!     // Read data
//!     let bs = o.read().await?;
//!
//!     // Fetch metadata
//!     let name = o.name();
//!     let path = o.path();
//!     let meta = o.metadata().await?;
//!     let mode = meta.mode();
//!     let length = meta.content_length();
//!
//!     // Delete
//!     o.delete().await?;
//!
//!     // Readdir
//!     let o = op.object("test_dir/");
//!     let mut ds = o.list().await?;
//!     while let Some(entry) = ds.try_next().await? {
//!         let path = entry.path();
//!         let mode = entry.mode();
//!     }
//!
//!     Ok(())
//! }
//! ```

// Make sure all our public APIs have docs.
#![warn(missing_docs)]
// Deny unused qualifications.
#![deny(unused_qualifications)]
// Add options below to allow/deny Clippy lints.

// Private module with public types, they will be accessed via `opendal::Xxxx`
mod accessor;
pub use accessor::Accessor;
pub use accessor::AccessorCapability;
pub use accessor::AccessorMetadata;

mod io;
pub use io::BlockingBytesRead;
pub use io::BlockingBytesReader;
pub use io::BytesRead;
pub use io::BytesReader;
pub use io::BytesSink;
pub use io::BytesStream;
pub use io::BytesWrite;
pub use io::BytesWriter;

mod operator;
pub use operator::BatchOperator;
pub use operator::Operator;

mod object;
pub use object::Object;
pub use object::ObjectEntry;
pub use object::ObjectIterator;
pub use object::ObjectMetadata;
pub use object::ObjectMode;
pub use object::ObjectMultipart;
pub use object::ObjectPart;
pub use object::ObjectReader;
pub use object::ObjectStreamer;

mod scheme;
pub use scheme::Scheme;

// Public modules, they will be accessed via `opendal::io_util::Xxxx`
pub mod http_util;
pub mod io_util;
pub mod layers;
pub use layers::Layer;
pub mod adapters;
pub mod ops;
pub mod services;

// Private modules, internal use only.
//
// Please don't export any type from this module.
mod error;
mod path;

#[cfg(test)]
mod tests {
    use std::mem::size_of;

    use super::*;

    /// This is not a real test case.
    ///
    /// We assert our public structs here to make sure we don't introduce
    /// unexpected struct/enum size change.
    #[test]
    fn assert_size() {
        assert_eq!(80, size_of::<AccessorMetadata>());
        assert_eq!(16, size_of::<Operator>());
        assert_eq!(16, size_of::<BatchOperator>());
        assert_eq!(56, size_of::<ObjectEntry>());
        assert_eq!(40, size_of::<Object>());
        assert_eq!(160, size_of::<ObjectMetadata>());
        assert_eq!(1, size_of::<ObjectMode>());
        assert_eq!(64, size_of::<ObjectMultipart>());
        assert_eq!(32, size_of::<ObjectPart>());
        assert_eq!(24, size_of::<Scheme>());
    }
}