opendal_obs/types/operator/builder.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::sync::Arc;
20
21use crate::layers::*;
22use crate::raw::*;
23use crate::*;
24
25/// # Operator build API
26///
27/// Operator should be built via [`OperatorBuilder`]. We recommend to use [`Operator::new`] to get started:
28///
29/// ```
30/// # use anyhow::Result;
31/// use opendal::services::Fs;
32/// use opendal::Operator;
33/// async fn test() -> Result<()> {
34/// // Create fs backend builder.
35/// let builder = Fs::default().root("/tmp");
36///
37/// // Build an `Operator` to start operating the storage.
38/// let op: Operator = Operator::new(builder)?.finish();
39///
40/// Ok(())
41/// }
42/// ```
43impl Operator {
44 /// Create a new operator with input builder.
45 ///
46 /// OpenDAL will call `builder.build()` internally, so we don't need
47 /// to import `opendal::Builder` trait.
48 ///
49 /// # Examples
50 ///
51 /// Read more backend init examples in [examples](https://github.com/apache/opendal/tree/main/examples).
52 ///
53 /// ```
54 /// # use anyhow::Result;
55 /// use opendal::services::Fs;
56 /// use opendal::Operator;
57 /// async fn test() -> Result<()> {
58 /// // Create fs backend builder.
59 /// let builder = Fs::default().root("/tmp");
60 ///
61 /// // Build an `Operator` to start operating the storage.
62 /// let op: Operator = Operator::new(builder)?.finish();
63 ///
64 /// Ok(())
65 /// }
66 /// ```
67 #[allow(clippy::new_ret_no_self)]
68 pub fn new<B: Builder>(ab: B) -> Result<OperatorBuilder<impl Access>> {
69 let acc = ab.build()?;
70 Ok(OperatorBuilder::new(acc))
71 }
72
73 /// Create a new operator from given config.
74 ///
75 /// # Examples
76 ///
77 /// ```
78 /// # use anyhow::Result;
79 /// use std::collections::HashMap;
80 ///
81 /// use opendal::services::MemoryConfig;
82 /// use opendal::Operator;
83 /// async fn test() -> Result<()> {
84 /// let cfg = MemoryConfig::default();
85 ///
86 /// // Build an `Operator` to start operating the storage.
87 /// let op: Operator = Operator::from_config(cfg)?.finish();
88 ///
89 /// Ok(())
90 /// }
91 /// ```
92 pub fn from_config<C: Configurator>(cfg: C) -> Result<OperatorBuilder<impl Access>> {
93 let builder = cfg.into_builder();
94 let acc = builder.build()?;
95 Ok(OperatorBuilder::new(acc))
96 }
97
98 /// Create a new operator from given iterator in static dispatch.
99 ///
100 /// # Notes
101 ///
102 /// `from_iter` generates a `OperatorBuilder` which allows adding layer in zero-cost way.
103 ///
104 /// # Examples
105 ///
106 /// ```
107 /// # use anyhow::Result;
108 /// use std::collections::HashMap;
109 ///
110 /// use opendal::services::Fs;
111 /// use opendal::Operator;
112 /// async fn test() -> Result<()> {
113 /// let map = HashMap::from([
114 /// // Set the root for fs, all operations will happen under this root.
115 /// //
116 /// // NOTE: the root must be absolute path.
117 /// ("root".to_string(), "/tmp".to_string()),
118 /// ]);
119 ///
120 /// // Build an `Operator` to start operating the storage.
121 /// let op: Operator = Operator::from_iter::<Fs>(map)?.finish();
122 ///
123 /// Ok(())
124 /// }
125 /// ```
126 #[allow(clippy::should_implement_trait)]
127 pub fn from_iter<B: Builder>(
128 iter: impl IntoIterator<Item = (String, String)>,
129 ) -> Result<OperatorBuilder<impl Access>> {
130 let builder = B::Config::from_iter(iter)?.into_builder();
131 let acc = builder.build()?;
132 Ok(OperatorBuilder::new(acc))
133 }
134
135 /// Create a new operator via given scheme and iterator of config value in dynamic dispatch.
136 ///
137 /// # Notes
138 ///
139 /// `via_iter` generates a `Operator` which allows building operator without generic type.
140 ///
141 /// # Examples
142 ///
143 /// ```
144 /// # use anyhow::Result;
145 /// use std::collections::HashMap;
146 ///
147 /// use opendal::Operator;
148 /// use opendal::Scheme;
149 /// async fn test() -> Result<()> {
150 /// let map = [
151 /// // Set the root for fs, all operations will happen under this root.
152 /// //
153 /// // NOTE: the root must be absolute path.
154 /// ("root".to_string(), "/tmp".to_string()),
155 /// ];
156 ///
157 /// // Build an `Operator` to start operating the storage.
158 /// let op: Operator = Operator::via_iter(Scheme::Fs, map)?;
159 ///
160 /// Ok(())
161 /// }
162 /// ```
163 #[allow(unused_variables, unreachable_code)]
164 pub fn via_iter(
165 scheme: Scheme,
166 iter: impl IntoIterator<Item = (String, String)>,
167 ) -> Result<Operator> {
168 let op = match scheme {
169 #[cfg(feature = "services-aliyun-drive")]
170 Scheme::AliyunDrive => Self::from_iter::<services::AliyunDrive>(iter)?.finish(),
171 #[cfg(feature = "services-atomicserver")]
172 Scheme::Atomicserver => Self::from_iter::<services::Atomicserver>(iter)?.finish(),
173 #[cfg(feature = "services-alluxio")]
174 Scheme::Alluxio => Self::from_iter::<services::Alluxio>(iter)?.finish(),
175 #[cfg(feature = "services-compfs")]
176 Scheme::Compfs => Self::from_iter::<services::Compfs>(iter)?.finish(),
177 #[cfg(feature = "services-upyun")]
178 Scheme::Upyun => Self::from_iter::<services::Upyun>(iter)?.finish(),
179 #[cfg(feature = "services-koofr")]
180 Scheme::Koofr => Self::from_iter::<services::Koofr>(iter)?.finish(),
181 #[cfg(feature = "services-yandex-disk")]
182 Scheme::YandexDisk => Self::from_iter::<services::YandexDisk>(iter)?.finish(),
183 #[cfg(feature = "services-pcloud")]
184 Scheme::Pcloud => Self::from_iter::<services::Pcloud>(iter)?.finish(),
185 #[cfg(feature = "services-chainsafe")]
186 Scheme::Chainsafe => Self::from_iter::<services::Chainsafe>(iter)?.finish(),
187 #[cfg(feature = "services-azblob")]
188 Scheme::Azblob => Self::from_iter::<services::Azblob>(iter)?.finish(),
189 #[cfg(feature = "services-azdls")]
190 Scheme::Azdls => Self::from_iter::<services::Azdls>(iter)?.finish(),
191 #[cfg(feature = "services-azfile")]
192 Scheme::Azfile => Self::from_iter::<services::Azfile>(iter)?.finish(),
193 #[cfg(feature = "services-b2")]
194 Scheme::B2 => Self::from_iter::<services::B2>(iter)?.finish(),
195 #[cfg(feature = "services-cacache")]
196 Scheme::Cacache => Self::from_iter::<services::Cacache>(iter)?.finish(),
197 #[cfg(feature = "services-cos")]
198 Scheme::Cos => Self::from_iter::<services::Cos>(iter)?.finish(),
199 #[cfg(feature = "services-d1")]
200 Scheme::D1 => Self::from_iter::<services::D1>(iter)?.finish(),
201 #[cfg(feature = "services-dashmap")]
202 Scheme::Dashmap => Self::from_iter::<services::Dashmap>(iter)?.finish(),
203 #[cfg(feature = "services-dropbox")]
204 Scheme::Dropbox => Self::from_iter::<services::Dropbox>(iter)?.finish(),
205 #[cfg(feature = "services-etcd")]
206 Scheme::Etcd => Self::from_iter::<services::Etcd>(iter)?.finish(),
207 #[cfg(feature = "services-foundationdb")]
208 Scheme::Foundationdb => Self::from_iter::<services::Foundationdb>(iter)?.finish(),
209 #[cfg(feature = "services-fs")]
210 Scheme::Fs => Self::from_iter::<services::Fs>(iter)?.finish(),
211 #[cfg(feature = "services-ftp")]
212 Scheme::Ftp => Self::from_iter::<services::Ftp>(iter)?.finish(),
213 #[cfg(feature = "services-gcs")]
214 Scheme::Gcs => Self::from_iter::<services::Gcs>(iter)?.finish(),
215 #[cfg(feature = "services-ghac")]
216 Scheme::Ghac => Self::from_iter::<services::Ghac>(iter)?.finish(),
217 #[cfg(feature = "services-gridfs")]
218 Scheme::Gridfs => Self::from_iter::<services::Gridfs>(iter)?.finish(),
219 #[cfg(feature = "services-github")]
220 Scheme::Github => Self::from_iter::<services::Github>(iter)?.finish(),
221 #[cfg(feature = "services-hdfs")]
222 Scheme::Hdfs => Self::from_iter::<services::Hdfs>(iter)?.finish(),
223 #[cfg(feature = "services-http")]
224 Scheme::Http => Self::from_iter::<services::Http>(iter)?.finish(),
225 #[cfg(feature = "services-huggingface")]
226 Scheme::Huggingface => Self::from_iter::<services::Huggingface>(iter)?.finish(),
227 #[cfg(feature = "services-ipfs")]
228 Scheme::Ipfs => Self::from_iter::<services::Ipfs>(iter)?.finish(),
229 #[cfg(feature = "services-ipmfs")]
230 Scheme::Ipmfs => Self::from_iter::<services::Ipmfs>(iter)?.finish(),
231 #[cfg(feature = "services-icloud")]
232 Scheme::Icloud => Self::from_iter::<services::Icloud>(iter)?.finish(),
233 #[cfg(feature = "services-libsql")]
234 Scheme::Libsql => Self::from_iter::<services::Libsql>(iter)?.finish(),
235 #[cfg(feature = "services-memcached")]
236 Scheme::Memcached => Self::from_iter::<services::Memcached>(iter)?.finish(),
237 #[cfg(feature = "services-memory")]
238 Scheme::Memory => Self::from_iter::<services::Memory>(iter)?.finish(),
239 #[cfg(feature = "services-mini-moka")]
240 Scheme::MiniMoka => Self::from_iter::<services::MiniMoka>(iter)?.finish(),
241 #[cfg(feature = "services-moka")]
242 Scheme::Moka => Self::from_iter::<services::Moka>(iter)?.finish(),
243 #[cfg(feature = "services-monoiofs")]
244 Scheme::Monoiofs => Self::from_iter::<services::Monoiofs>(iter)?.finish(),
245 #[cfg(feature = "services-mysql")]
246 Scheme::Mysql => Self::from_iter::<services::Mysql>(iter)?.finish(),
247 #[cfg(feature = "services-obs")]
248 Scheme::Obs => Self::from_iter::<services::Obs>(iter)?.finish(),
249 #[cfg(feature = "services-onedrive")]
250 Scheme::Onedrive => Self::from_iter::<services::Onedrive>(iter)?.finish(),
251 #[cfg(feature = "services-postgresql")]
252 Scheme::Postgresql => Self::from_iter::<services::Postgresql>(iter)?.finish(),
253 #[cfg(feature = "services-gdrive")]
254 Scheme::Gdrive => Self::from_iter::<services::Gdrive>(iter)?.finish(),
255 #[cfg(feature = "services-oss")]
256 Scheme::Oss => Self::from_iter::<services::Oss>(iter)?.finish(),
257 #[cfg(feature = "services-persy")]
258 Scheme::Persy => Self::from_iter::<services::Persy>(iter)?.finish(),
259 #[cfg(feature = "services-redis")]
260 Scheme::Redis => Self::from_iter::<services::Redis>(iter)?.finish(),
261 #[cfg(feature = "services-rocksdb")]
262 Scheme::Rocksdb => Self::from_iter::<services::Rocksdb>(iter)?.finish(),
263 #[cfg(feature = "services-s3")]
264 Scheme::S3 => Self::from_iter::<services::S3>(iter)?.finish(),
265 #[cfg(feature = "services-seafile")]
266 Scheme::Seafile => Self::from_iter::<services::Seafile>(iter)?.finish(),
267 #[cfg(feature = "services-sftp")]
268 Scheme::Sftp => Self::from_iter::<services::Sftp>(iter)?.finish(),
269 #[cfg(feature = "services-sled")]
270 Scheme::Sled => Self::from_iter::<services::Sled>(iter)?.finish(),
271 #[cfg(feature = "services-sqlite")]
272 Scheme::Sqlite => Self::from_iter::<services::Sqlite>(iter)?.finish(),
273 #[cfg(feature = "services-supabase")]
274 Scheme::Supabase => Self::from_iter::<services::Supabase>(iter)?.finish(),
275 #[cfg(feature = "services-swift")]
276 Scheme::Swift => Self::from_iter::<services::Swift>(iter)?.finish(),
277 #[cfg(feature = "services-tikv")]
278 Scheme::Tikv => Self::from_iter::<services::Tikv>(iter)?.finish(),
279 #[cfg(feature = "services-vercel-artifacts")]
280 Scheme::VercelArtifacts => Self::from_iter::<services::VercelArtifacts>(iter)?.finish(),
281 #[cfg(feature = "services-vercel-blob")]
282 Scheme::VercelBlob => Self::from_iter::<services::VercelBlob>(iter)?.finish(),
283 #[cfg(feature = "services-webdav")]
284 Scheme::Webdav => Self::from_iter::<services::Webdav>(iter)?.finish(),
285 #[cfg(feature = "services-webhdfs")]
286 Scheme::Webhdfs => Self::from_iter::<services::Webhdfs>(iter)?.finish(),
287 #[cfg(feature = "services-redb")]
288 Scheme::Redb => Self::from_iter::<services::Redb>(iter)?.finish(),
289 #[cfg(feature = "services-mongodb")]
290 Scheme::Mongodb => Self::from_iter::<services::Mongodb>(iter)?.finish(),
291 #[cfg(feature = "services-hdfs-native")]
292 Scheme::HdfsNative => Self::from_iter::<services::HdfsNative>(iter)?.finish(),
293 #[cfg(feature = "services-lakefs")]
294 Scheme::Lakefs => Self::from_iter::<services::Lakefs>(iter)?.finish(),
295 #[cfg(feature = "services-nebula-graph")]
296 Scheme::NebulaGraph => Self::from_iter::<services::NebulaGraph>(iter)?.finish(),
297 v => {
298 return Err(Error::new(
299 ErrorKind::Unsupported,
300 "scheme is not enabled or supported",
301 )
302 .with_context("scheme", v))
303 }
304 };
305
306 Ok(op)
307 }
308
309 /// Create a new operator from given map.
310 ///
311 /// # Notes
312 ///
313 /// from_map is using static dispatch layers which is zero cost. via_map is
314 /// using dynamic dispatch layers which has a bit runtime overhead with an
315 /// extra vtable lookup and unable to inline. But from_map requires generic
316 /// type parameter which is not always easy to be used.
317 ///
318 /// # Examples
319 ///
320 /// ```
321 /// # use anyhow::Result;
322 /// use std::collections::HashMap;
323 ///
324 /// use opendal::services::Fs;
325 /// use opendal::Operator;
326 /// async fn test() -> Result<()> {
327 /// let map = HashMap::from([
328 /// // Set the root for fs, all operations will happen under this root.
329 /// //
330 /// // NOTE: the root must be absolute path.
331 /// ("root".to_string(), "/tmp".to_string()),
332 /// ]);
333 ///
334 /// // Build an `Operator` to start operating the storage.
335 /// let op: Operator = Operator::from_map::<Fs>(map)?.finish();
336 ///
337 /// Ok(())
338 /// }
339 /// ```
340 #[deprecated = "use from_iter instead"]
341 pub fn from_map<B: Builder>(
342 map: HashMap<String, String>,
343 ) -> Result<OperatorBuilder<impl Access>> {
344 Self::from_iter::<B>(map)
345 }
346
347 /// Create a new operator from given scheme and map.
348 ///
349 /// # Notes
350 ///
351 /// from_map is using static dispatch layers which is zero cost. via_map is
352 /// using dynamic dispatch layers which has a bit runtime overhead with an
353 /// extra vtable lookup and unable to inline. But from_map requires generic
354 /// type parameter which is not always easy to be used.
355 ///
356 /// # Examples
357 ///
358 /// ```
359 /// # use anyhow::Result;
360 /// use std::collections::HashMap;
361 ///
362 /// use opendal::Operator;
363 /// use opendal::Scheme;
364 /// async fn test() -> Result<()> {
365 /// let map = HashMap::from([
366 /// // Set the root for fs, all operations will happen under this root.
367 /// //
368 /// // NOTE: the root must be absolute path.
369 /// ("root".to_string(), "/tmp".to_string()),
370 /// ]);
371 ///
372 /// // Build an `Operator` to start operating the storage.
373 /// let op: Operator = Operator::via_map(Scheme::Fs, map)?;
374 ///
375 /// Ok(())
376 /// }
377 /// ```
378 #[deprecated = "use via_iter instead"]
379 pub fn via_map(scheme: Scheme, map: HashMap<String, String>) -> Result<Operator> {
380 Self::via_iter(scheme, map)
381 }
382
383 /// Create a new layer with dynamic dispatch.
384 ///
385 /// # Notes
386 ///
387 /// `OperatorBuilder::layer()` is using static dispatch which is zero
388 /// cost. `Operator::layer()` is using dynamic dispatch which has a
389 /// bit runtime overhead with an extra vtable lookup and unable to
390 /// inline.
391 ///
392 /// It's always recommended to use `OperatorBuilder::layer()` instead.
393 ///
394 /// # Examples
395 ///
396 /// ```no_run
397 /// # use std::sync::Arc;
398 /// # use anyhow::Result;
399 /// use opendal::layers::LoggingLayer;
400 /// use opendal::services::Fs;
401 /// use opendal::Operator;
402 ///
403 /// # async fn test() -> Result<()> {
404 /// let op = Operator::new(Fs::default())?.finish();
405 /// let op = op.layer(LoggingLayer::default());
406 /// // All operations will go through the new_layer
407 /// let _ = op.read("test_file").await?;
408 /// # Ok(())
409 /// # }
410 /// ```
411 #[must_use]
412 pub fn layer<L: Layer<Accessor>>(self, layer: L) -> Self {
413 Self::from_inner(Arc::new(
414 TypeEraseLayer.layer(layer.layer(self.into_inner())),
415 ))
416 }
417}
418
419/// OperatorBuilder is a typed builder to build an Operator.
420///
421/// # Notes
422///
423/// OpenDAL uses static dispatch internally and only performs dynamic
424/// dispatch at the outmost type erase layer. OperatorBuilder is the only
425/// public API provided by OpenDAL come with generic parameters.
426///
427/// It's required to call `finish` after the operator built.
428///
429/// # Examples
430///
431/// For users who want to support many services, we can build a helper function like the following:
432///
433/// ```
434/// use std::collections::HashMap;
435///
436/// use opendal::layers::LoggingLayer;
437/// use opendal::layers::RetryLayer;
438/// use opendal::services;
439/// use opendal::Builder;
440/// use opendal::Operator;
441/// use opendal::Result;
442/// use opendal::Scheme;
443///
444/// fn init_service<B: Builder>(cfg: HashMap<String, String>) -> Result<Operator> {
445/// let op = Operator::from_map::<B>(cfg)?
446/// .layer(LoggingLayer::default())
447/// .layer(RetryLayer::new())
448/// .finish();
449///
450/// Ok(op)
451/// }
452///
453/// async fn init(scheme: Scheme, cfg: HashMap<String, String>) -> Result<()> {
454/// let _ = match scheme {
455/// Scheme::S3 => init_service::<services::S3>(cfg)?,
456/// Scheme::Fs => init_service::<services::Fs>(cfg)?,
457/// _ => todo!(),
458/// };
459///
460/// Ok(())
461/// }
462/// ```
463pub struct OperatorBuilder<A: Access> {
464 accessor: A,
465}
466
467impl<A: Access> OperatorBuilder<A> {
468 /// Create a new operator builder.
469 #[allow(clippy::new_ret_no_self)]
470 pub fn new(accessor: A) -> OperatorBuilder<impl Access> {
471 // Make sure error context layer has been attached.
472 OperatorBuilder { accessor }
473 .layer(ErrorContextLayer)
474 .layer(CompleteLayer)
475 }
476
477 /// Create a new layer with static dispatch.
478 ///
479 /// # Notes
480 ///
481 /// `OperatorBuilder::layer()` is using static dispatch which is zero
482 /// cost. `Operator::layer()` is using dynamic dispatch which has a
483 /// bit runtime overhead with an extra vtable lookup and unable to
484 /// inline.
485 ///
486 /// It's always recommended to use `OperatorBuilder::layer()` instead.
487 ///
488 /// # Examples
489 ///
490 /// ```no_run
491 /// # use std::sync::Arc;
492 /// # use anyhow::Result;
493 /// use opendal::layers::LoggingLayer;
494 /// use opendal::services::Fs;
495 /// use opendal::Operator;
496 ///
497 /// # async fn test() -> Result<()> {
498 /// let op = Operator::new(Fs::default())?
499 /// .layer(LoggingLayer::default())
500 /// .finish();
501 /// // All operations will go through the new_layer
502 /// let _ = op.read("test_file").await?;
503 /// # Ok(())
504 /// # }
505 /// ```
506 #[must_use]
507 pub fn layer<L: Layer<A>>(self, layer: L) -> OperatorBuilder<L::LayeredAccess> {
508 OperatorBuilder {
509 accessor: layer.layer(self.accessor),
510 }
511 }
512
513 /// Finish the building to construct an Operator.
514 pub fn finish(self) -> Operator {
515 let ob = self.layer(TypeEraseLayer);
516 Operator::from_inner(Arc::new(ob.accessor) as Accessor)
517 }
518}