pub struct Operator { /* private fields */ }
Expand description
Operator is the entry for all public async APIs.
Read concepts
for know more about Operator
.
Examples
Read more backend init examples in services
use opendal::services::Fs;
use opendal::Operator;
#[tokio::main]
async fn main() -> Result<()> {
// Create fs backend builder.
let mut builder = Fs::default();
// Set the root for fs, all operations will happen under this root.
//
// NOTE: the root must be absolute path.
builder.root("/tmp");
// Build an `Operator` to start operating the storage.
let _: Operator = Operator::new(builder)?.finish();
Ok(())
}
Implementations§
source§impl Operator
impl Operator
sourcepub fn with_limit(&self, limit: usize) -> Self
pub fn with_limit(&self, limit: usize) -> Self
Specify the batch limit.
Default: 1000
sourcepub fn info(&self) -> OperatorInfo
pub fn info(&self) -> OperatorInfo
sourcepub fn blocking(&self) -> BlockingOperator
pub fn blocking(&self) -> BlockingOperator
Create a new blocking operator.
This operation is nearly no cost.
source§impl Operator
impl Operator
Operator async API.
sourcepub async fn check(&self) -> Result<()>
pub async fn check(&self) -> Result<()>
Check if this operator can work correctly.
We will send a list
request to path and return any errors we met.
use opendal::Operator;
op.check().await?;
sourcepub async fn stat(&self, path: &str) -> Result<Metadata>
pub async fn stat(&self, path: &str) -> Result<Metadata>
Get current path’s metadata without cache directly.
Notes
Use stat
if you:
- Want detect the outside changes of path.
- Don’t want to read from cached metadata.
You may want to use metadata
if you are working with entries
returned by Lister
. It’s highly possible that metadata
you want has already been cached.
Examples
use opendal::ErrorKind;
if let Err(e) = op.stat("test").await {
if e.kind() == ErrorKind::NotFound {
println!("file not exist")
}
}
sourcepub async fn metadata(
&self,
entry: &Entry,
flags: impl Into<FlagSet<Metakey>>
) -> Result<Metadata>
pub async fn metadata( &self, entry: &Entry, flags: impl Into<FlagSet<Metakey>> ) -> Result<Metadata>
Get current metadata with cache.
metadata
will check the given query with already cached metadata
first. And query from storage if not found.
Notes
Use metadata
if you are working with entries returned by
Lister
. It’s highly possible that metadata you want
has already been cached.
You may want to use stat
, if you:
- Want detect the outside changes of path.
- Don’t want to read from cached metadata.
Behavior
Visiting not fetched metadata will lead to panic in debug build. It must be a bug, please fix it instead.
Examples
Query already cached metadata
By query metadata with None
, we can only query in-memory metadata
cache. In this way, we can make sure that no API call will send.
use opendal::Entry;
let meta = op.metadata(&entry, None).await?;
// content length COULD be correct.
let _ = meta.content_length();
// etag COULD be correct.
let _ = meta.etag();
Query content length and content type
use opendal::Entry;
use opendal::Metakey;
let meta = op
.metadata(&entry, Metakey::ContentLength | Metakey::ContentType)
.await?;
// content length MUST be correct.
let _ = meta.content_length();
// etag COULD be correct.
let _ = meta.etag();
Query all metadata
By query metadata with Complete
, we can make sure that we have fetched all metadata of this entry.
use opendal::Entry;
use opendal::Metakey;
let meta = op.metadata(&entry, Metakey::Complete).await?;
// content length MUST be correct.
let _ = meta.content_length();
// etag MUST be correct.
let _ = meta.etag();
sourcepub async fn is_exist(&self, path: &str) -> Result<bool>
pub async fn is_exist(&self, path: &str) -> Result<bool>
Check if this path exists or not.
Example
use anyhow::Result;
use futures::io;
use opendal::Operator;
#[tokio::main]
async fn test(op: Operator) -> Result<()> {
let _ = op.is_exist("test").await?;
Ok(())
}
sourcepub async fn create_dir(&self, path: &str) -> Result<()>
pub async fn create_dir(&self, path: &str) -> Result<()>
Create a dir at given path.
Notes
To indicate that a path is a directory, it is compulsory to include
a trailing / in the path. Failure to do so may result in
NotADirectory
error being returned by OpenDAL.
Behavior
- Create on existing dir will succeed.
- Create dir is always recursive, works like
mkdir -p
Examples
op.create_dir("path/to/dir/").await?;
sourcepub async fn read(&self, path: &str) -> Result<Vec<u8>>
pub async fn read(&self, path: &str) -> Result<Vec<u8>>
Read the whole path into a bytes.
This function will allocate a new bytes internally. For more precise memory control or
reading data lazily, please use Operator::reader
Examples
let bs = op.read("path/to/file").await?;
sourcepub async fn range_read(
&self,
path: &str,
range: impl RangeBounds<u64>
) -> Result<Vec<u8>>
pub async fn range_read( &self, path: &str, range: impl RangeBounds<u64> ) -> Result<Vec<u8>>
Read the specified range of path into a bytes.
This function will allocate a new bytes internally. For more precise memory control or
reading data lazily, please use Operator::range_reader
Notes
- The returning content’s length may be smaller than the range specified.
Examples
let bs = op.range_read("path/to/file", 1024..2048).await?;
sourcepub async fn reader(&self, path: &str) -> Result<Reader>
pub async fn reader(&self, path: &str) -> Result<Reader>
Create a new reader which can read the whole path.
Examples
let r = op.reader("path/to/file").await?;
sourcepub async fn range_reader(
&self,
path: &str,
range: impl RangeBounds<u64>
) -> Result<Reader>
pub async fn range_reader( &self, path: &str, range: impl RangeBounds<u64> ) -> Result<Reader>
sourcepub async fn write_with(
&self,
path: &str,
args: OpWrite,
bs: impl Into<Bytes>
) -> Result<()>
pub async fn write_with( &self, path: &str, args: OpWrite, bs: impl Into<Bytes> ) -> Result<()>
Write data with extra options.
Notes
- Write will make sure all bytes has been written, or an error will be returned.
Examples
use bytes::Bytes;
use opendal::ops::OpWrite;
let bs = b"hello, world!".to_vec();
let args = OpWrite::new().with_content_type("text/plain");
let _ = op.write_with("path/to/file", args, bs).await?;
sourcepub async fn remove_via(
&self,
input: impl Stream<Item = String> + Unpin
) -> Result<()>
pub async fn remove_via( &self, input: impl Stream<Item = String> + Unpin ) -> Result<()>
remove will given paths. remove_via will remove files via given stream.
We will delete by chunks with given batch limit on the stream.
Notes
If underlying services support delete in batch, we will use batch delete instead.
Examples
use futures::stream;
let stream = stream::iter(vec!["abc".to_string(), "def".to_string()]);
op.remove_via(stream).await?;
sourcepub async fn remove_all(&self, path: &str) -> Result<()>
pub async fn remove_all(&self, path: &str) -> Result<()>
sourcepub async fn list(&self, path: &str) -> Result<Lister>
pub async fn list(&self, path: &str) -> Result<Lister>
List given path.
This function will create a new handle to list entries.
An error will be returned if given path doesn’t end with /
.
Examples
use opendal::Metakey;
let mut ds = op.list("path/to/dir/").await?;
while let Some(mut de) = ds.try_next().await? {
let meta = op.metadata(&de, Metakey::Mode).await?;
match meta.mode() {
EntryMode::FILE => {
println!("Handling file")
}
EntryMode::DIR => {
println!("Handling dir like start a new list via meta.path()")
}
EntryMode::Unknown => continue,
}
}
sourcepub async fn scan(&self, path: &str) -> Result<Lister>
pub async fn scan(&self, path: &str) -> Result<Lister>
List dir in flat way.
This function will create a new handle to list entries.
An error will be returned if given path doesn’t end with /
.
Examples
use opendal::Metakey;
let mut ds = op.scan("/path/to/dir/").await?;
while let Some(mut de) = ds.try_next().await? {
let meta = op.metadata(&de, Metakey::Mode).await?;
match meta.mode() {
EntryMode::FILE => {
println!("Handling file")
}
EntryMode::DIR => {
println!("Handling dir like start a new list via meta.path()")
}
EntryMode::Unknown => continue,
}
}
source§impl Operator
impl Operator
Operator presign API.
sourcepub fn presign_stat(
&self,
path: &str,
expire: Duration
) -> Result<PresignedRequest>
pub fn presign_stat( &self, path: &str, expire: Duration ) -> Result<PresignedRequest>
Presign an operation for stat(head).
Example
use anyhow::Result;
use futures::io;
use opendal::Operator;
use time::Duration;
#[tokio::main]
async fn test(op: Operator) -> Result<()> {
let signed_req = op.presign_stat("test",Duration::hours(1))?;
let req = http::Request::builder()
.method(signed_req.method())
.uri(signed_req.uri())
.body(())?;
sourcepub fn presign_read(
&self,
path: &str,
expire: Duration
) -> Result<PresignedRequest>
pub fn presign_read( &self, path: &str, expire: Duration ) -> Result<PresignedRequest>
Presign an operation for read.
Example
use anyhow::Result;
use futures::io;
use opendal::Operator;
use time::Duration;
#[tokio::main]
async fn test(op: Operator) -> Result<()> {
let signed_req = op.presign_read("test.txt", Duration::hours(1))?;
signed_req.method()
:GET
signed_req.uri()
:https://s3.amazonaws.com/examplebucket/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=access_key_id/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=<signature-value>
signed_req.headers()
:{ "host": "s3.amazonaws.com" }
We can download this file via curl
or other tools without credentials:
curl "https://s3.amazonaws.com/examplebucket/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=access_key_id/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=<signature-value>" -O /tmp/test.txt
sourcepub fn presign_write(
&self,
path: &str,
expire: Duration
) -> Result<PresignedRequest>
pub fn presign_write( &self, path: &str, expire: Duration ) -> Result<PresignedRequest>
Presign an operation for write.
Example
use anyhow::Result;
use futures::io;
use opendal::Operator;
use time::Duration;
#[tokio::main]
async fn test(op: Operator) -> Result<()> {
let signed_req = op.presign_write("test.txt", Duration::hours(1))?;
signed_req.method()
:PUT
signed_req.uri()
:https://s3.amazonaws.com/examplebucket/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=access_key_id/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=<signature-value>
signed_req.headers()
:{ "host": "s3.amazonaws.com" }
We can upload file as this file via curl
or other tools without credential:
curl -X PUT "https://s3.amazonaws.com/examplebucket/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=access_key_id/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=<signature-value>" -d "Hello, World!"
sourcepub fn presign_write_with(
&self,
path: &str,
op: OpWrite,
expire: Duration
) -> Result<PresignedRequest>
pub fn presign_write_with( &self, path: &str, op: OpWrite, expire: Duration ) -> Result<PresignedRequest>
Presign an operation for write with option described in OpenDAL rfc-0661
You can pass OpWrite
to this method to specify the content length and content type.
Example
use anyhow::Result;
use futures::io;
use opendal::ops::OpWrite;
use opendal::Operator;
use time::Duration;
#[tokio::main]
async fn test(op: Operator) -> Result<()> {
let args = OpWrite::new().with_content_type("text/csv");
let signed_req = op.presign_write_with("test", args, Duration::hours(1))?;
let req = http::Request::builder()
.method(signed_req.method())
.uri(signed_req.uri())
.body(())?;
source§impl Operator
impl Operator
Operator build API
Operator should be built via OperatorBuilder
. We recommend to use [Operator::create
] to get started:
use opendal::services::Fs;
use opendal::Operator;
#[tokio::main]
async fn main() -> Result<()> {
// Create fs backend builder.
let mut builder = Fs::default();
// Set the root for fs, all operations will happen under this root.
//
// NOTE: the root must be absolute path.
builder.root("/tmp");
// Build an `Operator` to start operating the storage.
let op: Operator = Operator::new(builder)?.finish();
Ok(())
}
sourcepub fn new<B: Builder>(ab: B) -> Result<OperatorBuilder<impl Accessor>>
pub fn new<B: Builder>(ab: B) -> Result<OperatorBuilder<impl Accessor>>
Create a new operator with input builder.
OpenDAL will call builder.build()
internally, so we don’t need
to import opendal::Builder
trait.
Examples
Read more backend init examples in examples.
use opendal::services::Fs;
use opendal::Operator;
#[tokio::main]
async fn main() -> Result<()> {
// Create fs backend builder.
let mut builder = Fs::default();
// Set the root for fs, all operations will happen under this root.
//
// NOTE: the root must be absolute path.
builder.root("/tmp");
// Build an `Operator` to start operating the storage.
let op: Operator = Operator::new(builder)?.finish();
Ok(())
}
sourcepub fn from_map<B: Builder>(
map: HashMap<String, String>
) -> Result<OperatorBuilder<impl Accessor>>
pub fn from_map<B: Builder>( map: HashMap<String, String> ) -> Result<OperatorBuilder<impl Accessor>>
Create a new operator from given map.
use std::collections::HashMap;
use opendal::services::Fs;
use opendal::Operator;
#[tokio::main]
async fn main() -> Result<()> {
let map = HashMap::from([
// Set the root for fs, all operations will happen under this root.
//
// NOTE: the root must be absolute path.
("root".to_string(), "/tmp".to_string()),
]);
// Build an `Operator` to start operating the storage.
let op: Operator = Operator::from_map::<Fs>(map)?.finish();
Ok(())
}
sourcepub fn from_iter<B: Builder>(
iter: impl Iterator<Item = (String, String)>
) -> Result<OperatorBuilder<impl Accessor>>
pub fn from_iter<B: Builder>( iter: impl Iterator<Item = (String, String)> ) -> Result<OperatorBuilder<impl Accessor>>
Create a new operator from iter.
WARNING
It’s better to use from_map
. We may remove this API in the
future.
sourcepub fn from_env<B: Builder>() -> Result<OperatorBuilder<impl Accessor>>
pub fn from_env<B: Builder>() -> Result<OperatorBuilder<impl Accessor>>
Create a new operator from env.
WARNING
It’s better to use from_map
. We may remove this API in the
future.
sourcepub fn layer<L: Layer<FusedAccessor>>(self, layer: L) -> Self
pub fn layer<L: Layer<FusedAccessor>>(self, layer: L) -> Self
Create a new layer with dynamic dispatch.
Notes
OperatorBuilder::layer()
is using static dispatch which is zero
cost. Operator::layer()
is using dynamic dispatch which has a
bit runtime overhead with an extra vtable lookup and unable to
inline.
It’s always recommended to use OperatorBuilder::layer()
instead.
Examples
use opendal::layers::LoggingLayer;
use opendal::services::Fs;
use opendal::Operator;
let op = Operator::new(Fs::default())?.finish();
let op = op.layer(LoggingLayer::default());
// All operations will go through the new_layer
let _ = op.read("test_file").await?;