remotefs_aws_s3/
client.rs

1//! # Aws s3
2//!
3//! Aws s3 client for remotefs
4
5use std::io::Read;
6use std::path::{Path, PathBuf};
7use std::sync::Arc;
8
9use aws_config::Region;
10use aws_config::default_provider::credentials::DefaultCredentialsChain;
11use aws_config::meta::region::RegionProviderChain;
12pub use aws_sdk_s3::Client as S3Client;
13use aws_sdk_s3::config::{Builder as S3ClientBuilder, Credentials, ProvideCredentials};
14use aws_sdk_s3::primitives::{ByteStream, SdkBody};
15use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Object};
16#[cfg(target_os = "windows")]
17use path_slash::PathExt as _;
18use remotefs::fs::{Metadata, ReadStream, UnixPex, Welcome, WriteStream};
19use remotefs::{File, RemoteError, RemoteErrorType, RemoteFs, RemoteResult};
20use tokio::runtime::Runtime;
21
22use super::object::S3Object;
23use crate::utils::path as path_utils;
24
25const MIN_MULTIPART_UPLOAD_SIZE: usize = 5 * 1024 * 1024; // 5MB
26
27/// Aws s3 file system client
28pub struct AwsS3Fs {
29    client: Option<S3Client>,
30    runtime: Arc<Runtime>,
31    wrkdir: PathBuf,
32    // -- options
33    bucket_name: String,
34    /// Region name, if unset `Custom`.
35    region: Option<String>,
36    /// Custom endpoint (useful for minio)
37    endpoint: Option<String>,
38    profile: Option<String>,
39    access_key: Option<String>,
40    secret_key: Option<String>,
41    security_token: Option<String>,
42    session_token: Option<String>,
43    /// New path style. Required for some backends, such as MinIO
44    new_path_style: bool,
45}
46
47#[derive(Debug)]
48pub enum RemoteFsCredentials {
49    Default(DefaultCredentialsChain),
50    User(Credentials),
51}
52
53impl ProvideCredentials for RemoteFsCredentials {
54    fn fallback_on_interrupt(&self) -> Option<Credentials> {
55        match self {
56            Self::Default(c) => c.fallback_on_interrupt(),
57            Self::User(c) => c.fallback_on_interrupt(),
58        }
59    }
60
61    fn provide_credentials<'a>(
62        &'a self,
63    ) -> aws_credential_types::provider::future::ProvideCredentials<'a>
64    where
65        Self: 'a,
66    {
67        match self {
68            Self::Default(c) => c.provide_credentials(),
69            Self::User(c) => c.provide_credentials(),
70        }
71    }
72}
73
74impl AwsS3Fs {
75    /// Initialize a new `AwsS3Fs`
76    pub fn new<S: AsRef<str>>(bucket: S, runtime: &Arc<Runtime>) -> Self {
77        Self {
78            client: None,
79            runtime: runtime.clone(),
80            wrkdir: PathBuf::from("/"),
81            bucket_name: bucket.as_ref().to_string(),
82            region: None,
83            endpoint: None,
84            profile: None,
85            access_key: None,
86            secret_key: None,
87            security_token: None,
88            session_token: None,
89            new_path_style: false,
90        }
91    }
92
93    /// Specify region to connect to
94    pub fn region<S: AsRef<str>>(mut self, region: S) -> Self {
95        self.region = Some(region.as_ref().to_string());
96        self
97    }
98
99    /// Specify custom endpoint
100    /// This should be used when trying to connect to `minio` or other API compatible endpoints.
101    pub fn endpoint<S: AsRef<str>>(mut self, endpoint: S) -> Self {
102        self.endpoint = Some(endpoint.as_ref().to_string());
103        self
104    }
105
106    /// Set aws profile. If unset, "default" will be used
107    pub fn profile<S: AsRef<str>>(mut self, profile: S) -> Self {
108        self.profile = Some(profile.as_ref().to_string());
109        self
110    }
111
112    /// Set whether to use new path style. Required for backends such as MinIO (Default: False)
113    pub fn new_path_style(mut self, new_path_style: bool) -> Self {
114        self.new_path_style = new_path_style;
115        self
116    }
117
118    /// Specify access key for aws connection.
119    /// If unset, will be read from environment variable `AWS_ACCESS_KEY_ID`
120    pub fn access_key<S: AsRef<str>>(mut self, key: S) -> Self {
121        self.access_key = Some(key.as_ref().to_string());
122        self
123    }
124
125    /// Specify secret access key for aws connection.
126    /// If unset, will be read from environment variable `AWS_SECRET_ACCESS_KEY`
127    pub fn secret_access_key<S: AsRef<str>>(mut self, key: S) -> Self {
128        self.secret_key = Some(key.as_ref().to_string());
129        self
130    }
131
132    /// Specify security token for aws connection.
133    /// If unset, will be read from environment variable `AWS_SECURITY_TOKEN`
134    pub fn security_token<S: AsRef<str>>(mut self, key: S) -> Self {
135        self.security_token = Some(key.as_ref().to_string());
136        self
137    }
138
139    /// Specify session token for aws connection.
140    /// If unset, will be read from environment variable `AWS_SESSION_TOKEN`
141    pub fn session_token<S: AsRef<str>>(mut self, key: S) -> Self {
142        self.session_token = Some(key.as_ref().to_string());
143        self
144    }
145
146    // -- get ref
147
148    /// Get a reference to the underlying aws sdk [`S3Client`] struct
149    pub fn client(&self) -> Option<&S3Client> {
150        self.client.as_ref()
151    }
152
153    // -- private
154
155    /// List objects contained in `p` path
156    fn list_objects(&self, p: &Path, list_dir: bool) -> RemoteResult<Vec<S3Object>> {
157        // Make path relative
158        let key: String = Self::fmt_path(p, list_dir);
159        debug!("Query list directory {}; key: {}", p.display(), key);
160        self.query_objects(key, true)
161    }
162
163    /// Stat an s3 object
164    fn stat_object(&self, p: &Path) -> RemoteResult<S3Object> {
165        let key: String = Self::fmt_path(p, false);
166        debug!("Query stat object {}; key: {}", p.display(), key);
167        let objects = self.query_objects(key, false)?;
168        // Absolutize path
169        let absol: PathBuf = path_utils::absolutize(Path::new("/"), p);
170        // Find associated object
171        match objects
172            .into_iter()
173            .find(|x| x.path.as_path() == absol.as_path())
174        {
175            Some(obj) => Ok(obj),
176            None => Err(RemoteError::new_ex(
177                RemoteErrorType::NoSuchFileOrDirectory,
178                format!("{}: No such file or directory", p.display()),
179            )),
180        }
181    }
182
183    /// Query objects at key
184    fn query_objects(
185        &self,
186        key: String,
187        only_direct_children: bool,
188    ) -> RemoteResult<Vec<S3Object>> {
189        debug!("query objects with prefix: '{key}'");
190
191        let fut = self
192            .client
193            .as_ref()
194            .unwrap()
195            .list_objects_v2()
196            .bucket(self.bucket_name.as_str())
197            .prefix(key.as_str())
198            .send();
199        let results = self.runtime.block_on(fut);
200        match results {
201            Ok(entries) => {
202                let Some(contents) = entries.contents else {
203                    debug!("No objects found at key {key}",);
204                    return Ok(vec![]);
205                };
206
207                let objects: Vec<S3Object> = contents
208                    .into_iter()
209                    .filter(|object| {
210                        if only_direct_children {
211                            Self::list_object_should_be_kept(object, key.as_str())
212                        } else {
213                            true
214                        }
215                    })
216                    .map(S3Object::from)
217                    .collect();
218
219                debug!("Found objects: {:?}", objects);
220                Ok(objects)
221            }
222            Err(e) => Err(RemoteError::new_ex(RemoteErrorType::StatFailed, e)),
223        }
224    }
225
226    /// Returns whether object should be kept after list command.
227    /// The object won't be kept if:
228    ///
229    /// 1. is not a direct child of provided dir
230    fn list_object_should_be_kept(obj: &Object, dir: &str) -> bool {
231        Self::is_direct_child(obj.key.as_deref().unwrap_or_default(), dir)
232    }
233
234    /// Checks whether Object's key is direct child of `parent` path.
235    fn is_direct_child(key: &str, parent: &str) -> bool {
236        key == format!("{}{}", parent, S3Object::object_name(key))
237            || key == format!("{}{}/", parent, S3Object::object_name(key))
238    }
239
240    /// Make s3 absolute path from a given path
241    fn resolve(&self, p: &Path) -> PathBuf {
242        path_utils::diff_paths(
243            path_utils::absolutize(self.wrkdir.as_path(), p),
244            Path::new("/"),
245        )
246        .unwrap_or_default()
247    }
248
249    /// fmt path for fsentry according to format expected by s3
250    fn fmt_path(p: &Path, is_dir: bool) -> String {
251        // prevent root as slash
252        if p == Path::new("/") {
253            return "".to_string();
254        }
255        // Remove root only if absolute
256        #[cfg(target_family = "unix")]
257        let is_absolute: bool = p.is_absolute();
258        // NOTE: don't use is_absolute: on windows won't work
259        #[cfg(target_family = "windows")]
260        let is_absolute: bool = p.display().to_string().starts_with('/');
261        let p: PathBuf = match is_absolute {
262            true => path_utils::diff_paths(p, Path::new("/")).unwrap_or_default(),
263            false => p.to_path_buf(),
264        };
265        // NOTE: windows only: resolve paths
266        #[cfg(target_family = "windows")]
267        let p: PathBuf = PathBuf::from(p.to_slash_lossy().to_string());
268        // Fmt
269        match is_dir {
270            true => {
271                let mut p: String = p.display().to_string();
272                if !p.ends_with('/') {
273                    p.push('/');
274                }
275                p
276            }
277            false => p.to_string_lossy().to_string(),
278        }
279    }
280
281    /// Check connection status
282    fn check_connection(&mut self) -> RemoteResult<()> {
283        if self.is_connected() {
284            Ok(())
285        } else {
286            Err(RemoteError::new(RemoteErrorType::NotConnected))
287        }
288    }
289
290    /// Return whether connection should use anonymous credentials
291    fn is_anonymous(&self) -> bool {
292        self.access_key.is_none()
293            && self.secret_key.is_none()
294            && self.security_token.is_none()
295            && self.session_token.is_none()
296    }
297
298    /// Load credentials for current session
299    fn load_credentials(&self, region: RegionProviderChain) -> RemoteResult<RemoteFsCredentials> {
300        if self.is_anonymous() {
301            Ok(RemoteFsCredentials::Default(self.runtime.block_on(
302                DefaultCredentialsChain::builder().region(region).build(),
303            )))
304        } else {
305            let Some(access_key) = self.access_key.as_ref() else {
306                return Err(RemoteError::new_ex(
307                    RemoteErrorType::AuthenticationFailed,
308                    "Access key not set",
309                ));
310            };
311            let Some(secret_key) = self.secret_key.as_ref() else {
312                return Err(RemoteError::new_ex(
313                    RemoteErrorType::AuthenticationFailed,
314                    "Secret key not set",
315                ));
316            };
317
318            Ok(RemoteFsCredentials::User(Credentials::new(
319                access_key,
320                secret_key,
321                self.session_token.clone(),
322                None,
323                "default",
324            )))
325        }
326    }
327
328    /// Initialize region to be used from connection parameters
329    fn init_region(&self) -> RemoteResult<RegionProviderChain> {
330        Ok(
331            RegionProviderChain::first_try(self.region.as_ref().cloned().map(Region::new))
332                .or_default_provider()
333                .or_else(Region::new("us-west-2")),
334        )
335    }
336
337    /// Make bucket based on current options
338    fn make_client(
339        &self,
340        region: Option<Region>,
341        credentials: impl ProvideCredentials + 'static,
342    ) -> S3Client {
343        let mut builder = S3ClientBuilder::new()
344            .credentials_provider(credentials)
345            .behavior_version_latest()
346            .region(region);
347        builder.set_force_path_style(Some(self.new_path_style));
348
349        builder.set_endpoint_url(self.endpoint.clone());
350
351        S3Client::from_conf(builder.build())
352    }
353}
354
355impl RemoteFs for AwsS3Fs {
356    fn connect(&mut self) -> RemoteResult<Welcome> {
357        // Load credentials
358        debug!("Loading credentials... (profile {:?})", self.profile);
359        let region_provider = self.init_region()?;
360        let region = self.runtime.block_on(region_provider.region());
361        let credentials = self.load_credentials(region_provider)?;
362        // Parse region
363        trace!(
364            "Parsing region: {}; endpoint: {}",
365            self.region.as_deref().unwrap_or("NULL"),
366            self.endpoint.as_deref().unwrap_or("NULL")
367        );
368        debug!(
369            "Credentials loaded! Connecting to bucket {}...",
370            self.bucket_name
371        );
372        self.client = Some(self.make_client(region, credentials));
373        info!("Connection successfully established to s3 bucket");
374        Ok(Welcome::default())
375    }
376
377    fn disconnect(&mut self) -> RemoteResult<()> {
378        info!("Disconnecting from S3 bucket...");
379        match self.client.take() {
380            Some(bucket) => {
381                drop(bucket);
382                Ok(())
383            }
384            None => Err(RemoteError::new(RemoteErrorType::NotConnected)),
385        }
386    }
387
388    fn is_connected(&mut self) -> bool {
389        self.client.is_some()
390    }
391
392    fn pwd(&mut self) -> RemoteResult<PathBuf> {
393        self.check_connection()?;
394        Ok(self.wrkdir.clone())
395    }
396
397    fn change_dir(&mut self, dir: &Path) -> RemoteResult<PathBuf> {
398        self.check_connection()?;
399        // Always allow entering root
400        if dir == Path::new("/") {
401            self.wrkdir = dir.to_path_buf();
402            debug!("New working directory: {}", self.wrkdir.display());
403            return Ok(self.wrkdir.clone());
404        }
405        // Check if directory exists
406        debug!("Entering directory {}...", dir.display());
407        let dir_p: PathBuf = self.resolve(dir);
408        let dir_s: String = Self::fmt_path(dir_p.as_path(), true);
409        debug!("Searching for key {} (path: {})...", dir_s, dir_p.display());
410        // Check if directory already exists
411        if self
412            .stat_object(PathBuf::from(dir_s.as_str()).as_path())
413            .is_ok()
414        {
415            self.wrkdir = path_utils::absolutize(Path::new("/"), dir_p.as_path());
416            debug!("New working directory: {}", self.wrkdir.display());
417            Ok(self.wrkdir.clone())
418        } else {
419            Err(RemoteError::new(RemoteErrorType::NoSuchFileOrDirectory))
420        }
421    }
422
423    fn list_dir(&mut self, path: &Path) -> RemoteResult<Vec<File>> {
424        self.check_connection()?;
425        self.list_objects(path, true)
426            .map(|x| x.into_iter().map(|x| x.into()).collect())
427    }
428
429    fn stat(&mut self, path: &Path) -> RemoteResult<File> {
430        self.check_connection()?;
431        let path = self.resolve(path);
432        if let Ok(obj) = self.stat_object(path.as_path()) {
433            return Ok(obj.into());
434        }
435        // Try as a "directory"
436        trace!("Failed to stat object as file; trying as a directory...");
437        let path = PathBuf::from(Self::fmt_path(path.as_path(), true));
438        self.stat_object(path.as_path()).map(|x| x.into())
439    }
440
441    fn setstat(&mut self, _path: &Path, _metadata: Metadata) -> RemoteResult<()> {
442        Err(RemoteError::new(RemoteErrorType::UnsupportedFeature))
443    }
444
445    fn exists(&mut self, path: &Path) -> RemoteResult<bool> {
446        match self.stat(path) {
447            Ok(_) => Ok(true),
448            Err(RemoteError {
449                kind: RemoteErrorType::NoSuchFileOrDirectory,
450                ..
451            }) => Ok(false),
452            Err(err) => Err(err),
453        }
454    }
455
456    fn remove_file(&mut self, path: &Path) -> RemoteResult<()> {
457        self.check_connection()?;
458        let path = Self::fmt_path(self.resolve(path).as_path(), false);
459        debug!("Removing object '{}'", path);
460        let fut = self
461            .client
462            .as_ref()
463            .unwrap()
464            .delete_object()
465            .bucket(self.bucket_name.as_str())
466            .key(path.as_str())
467            .send();
468
469        self.runtime
470            .block_on(fut)
471            .map_err(|e| {
472                RemoteError::new_ex(
473                    RemoteErrorType::ProtocolError,
474                    format!("Could not remove file: {}", e),
475                )
476            })
477            .map(|_| ())
478    }
479
480    fn remove_dir(&mut self, path: &Path) -> RemoteResult<()> {
481        self.check_connection()?;
482        if !self.exists(path).ok().unwrap_or(false) {
483            return Err(RemoteError::new(RemoteErrorType::NoSuchFileOrDirectory));
484        }
485        println!("{}", self.resolve(path).as_path().display());
486        let path = Self::fmt_path(self.resolve(path).as_path(), true);
487        debug!("Removing object {}...", path);
488        let fut = self
489            .client
490            .as_ref()
491            .unwrap()
492            .delete_object()
493            .bucket(self.bucket_name.as_str())
494            .key(path.as_str())
495            .send();
496
497        self.runtime.block_on(fut).map(|_| ()).map_err(|e| {
498            RemoteError::new_ex(
499                RemoteErrorType::ProtocolError,
500                format!("Could not remove directory: {}", e),
501            )
502        })
503    }
504
505    fn remove_dir_all(&mut self, path: &Path) -> RemoteResult<()> {
506        debug!("Removing all content of {}", path.display());
507        if self.remove_dir(path).is_err() {
508            self.remove_file(path)
509        } else {
510            Ok(())
511        }
512    }
513
514    fn create_dir(&mut self, path: &Path, _mode: UnixPex) -> RemoteResult<()> {
515        self.check_connection()?;
516        let dir: String = Self::fmt_path(self.resolve(path).as_path(), true);
517        debug!("Making directory {}...", dir);
518        // Check if directory already exists
519        if self
520            .stat_object(PathBuf::from(dir.as_str()).as_path())
521            .is_ok()
522        {
523            error!("Directory {} already exists", dir);
524            return Err(RemoteError::new(RemoteErrorType::DirectoryAlreadyExists));
525        }
526        let fut = self
527            .client
528            .as_ref()
529            .unwrap()
530            .put_object()
531            .bucket(self.bucket_name.as_str())
532            .key(dir.as_str())
533            .send();
534
535        self.runtime.block_on(fut).map(|_| ()).map_err(|e| {
536            RemoteError::new_ex(
537                RemoteErrorType::FileCreateDenied,
538                format!("Could not make directory: {}", e),
539            )
540        })
541    }
542
543    fn symlink(&mut self, _path: &Path, _target: &Path) -> RemoteResult<()> {
544        Err(RemoteError::new(RemoteErrorType::UnsupportedFeature))
545    }
546
547    fn copy(&mut self, _src: &Path, _dest: &Path) -> RemoteResult<()> {
548        Err(RemoteError::new(RemoteErrorType::UnsupportedFeature))
549    }
550
551    fn mov(&mut self, _src: &Path, _dest: &Path) -> RemoteResult<()> {
552        Err(RemoteError::new(RemoteErrorType::UnsupportedFeature))
553    }
554
555    fn exec(&mut self, _cmd: &str) -> RemoteResult<(u32, String)> {
556        Err(RemoteError::new(RemoteErrorType::UnsupportedFeature))
557    }
558
559    fn append(&mut self, _path: &Path, _metadata: &Metadata) -> RemoteResult<WriteStream> {
560        Err(RemoteError::new(RemoteErrorType::UnsupportedFeature))
561    }
562
563    fn create(&mut self, _path: &Path, _metadata: &Metadata) -> RemoteResult<WriteStream> {
564        Err(RemoteError::new(RemoteErrorType::UnsupportedFeature))
565    }
566
567    fn open(&mut self, _path: &Path) -> RemoteResult<ReadStream> {
568        Err(RemoteError::new(RemoteErrorType::UnsupportedFeature))
569    }
570
571    fn create_file(
572        &mut self,
573        path: &Path,
574        _metadata: &Metadata,
575        mut reader: Box<dyn Read + Send>,
576    ) -> RemoteResult<u64> {
577        self.check_connection()?;
578        let src = self.resolve(path);
579        let key = Self::fmt_path(src.as_path(), false);
580
581        // we write 4096 bytes at a time
582        debug!("Query PUT for key '{}'", key);
583        let mut buf = vec![0; MIN_MULTIPART_UPLOAD_SIZE];
584        let mut offset = 0;
585        let mut part_number = 1;
586
587        // create multipart upload
588        let fut = self
589            .client
590            .as_ref()
591            .unwrap()
592            .create_multipart_upload()
593            .bucket(self.bucket_name.as_str())
594            .key(key.as_str())
595            .send();
596
597        let upload = self.runtime.block_on(fut).map_err(|e| {
598            error!("Could not init multipart upload: {e:?}",);
599            RemoteError::new_ex(
600                RemoteErrorType::ProtocolError,
601                format!("Could not init multipart upload: {}", e),
602            )
603        })?;
604        let upload_id = upload.upload_id().ok_or(RemoteError::new_ex(
605            RemoteErrorType::ProtocolError,
606            "no multipart id",
607        ))?;
608        debug!("starting multipart upload with upload id {upload_id}");
609
610        let mut upload_parts: Vec<aws_sdk_s3::types::CompletedPart> = Vec::new();
611
612        loop {
613            let buflen = reader.read(&mut buf).map_err(|e| {
614                RemoteError::new_ex(
615                    RemoteErrorType::IoError,
616                    format!("Could not read file: {}", e),
617                )
618            })?;
619            if buflen == 0 {
620                break;
621            }
622
623            let bytestream = ByteStream::new(SdkBody::from(&buf[..buflen]));
624            debug!("sending part {part_number} for {key} at offset {offset} for {buflen} bytes");
625
626            let buflen = buflen as i64;
627
628            let fut = self
629                .client
630                .as_ref()
631                .unwrap()
632                .upload_part()
633                .bucket(self.bucket_name.as_str())
634                .key(key.as_str())
635                .upload_id(upload_id)
636                .body(bytestream)
637                .part_number(part_number)
638                .send();
639
640            let upload_part_res = self.runtime.block_on(fut).map_err(|e| {
641                error!("Could not put file: {e:?}",);
642                RemoteError::new_ex(
643                    RemoteErrorType::ProtocolError,
644                    format!("Could not put file: {}", e),
645                )
646            })?;
647
648            upload_parts.push(
649                CompletedPart::builder()
650                    .e_tag(upload_part_res.e_tag.unwrap_or_default())
651                    .part_number(part_number)
652                    .build(),
653            );
654
655            debug!("written {buflen} bytes at offset {offset} for {key}");
656
657            offset += buflen;
658            part_number += 1;
659        }
660
661        // complete multipart
662        let completed_multipart_upload: CompletedMultipartUpload =
663            CompletedMultipartUpload::builder()
664                .set_parts(Some(upload_parts))
665                .build();
666
667        let fut = self
668            .client
669            .as_ref()
670            .unwrap()
671            .complete_multipart_upload()
672            .bucket(self.bucket_name.as_str())
673            .key(key.as_str())
674            .upload_id(upload_id)
675            .multipart_upload(completed_multipart_upload)
676            .send();
677
678        self.runtime.block_on(fut).map_err(|e| {
679            error!("Could not complete multipart upload: {e:?}",);
680            RemoteError::new_ex(
681                RemoteErrorType::ProtocolError,
682                format!("Could not complete multipart upload: {}", e),
683            )
684        })?;
685
686        Ok(offset as u64)
687    }
688
689    fn open_file(
690        &mut self,
691        src: &Path,
692        mut dest: Box<dyn std::io::Write + Send>,
693    ) -> RemoteResult<u64> {
694        self.check_connection()?;
695        if !self.exists(src).ok().unwrap_or(false) {
696            return Err(RemoteError::new(RemoteErrorType::NoSuchFileOrDirectory));
697        }
698        let src = self.resolve(src);
699        let key = Self::fmt_path(src.as_path(), false);
700        info!("Query GET for key '{}'", key);
701
702        let fut = self
703            .client
704            .as_ref()
705            .unwrap()
706            .get_object()
707            .bucket(self.bucket_name.as_str())
708            .key(key)
709            .send();
710
711        let mut data = self
712            .runtime
713            .block_on(fut)
714            .map_err(|e| {
715                RemoteError::new_ex(
716                    RemoteErrorType::ProtocolError,
717                    format!("Could not get file: {}", e),
718                )
719            })?
720            .body;
721
722        let mut n = 0;
723
724        while let Some(bytes) = self.runtime.block_on(data.try_next()).map_err(|e| {
725            RemoteError::new_ex(
726                RemoteErrorType::ProtocolError,
727                format!("Could not get file: {}", e),
728            )
729        })? {
730            dest.write_all(&bytes).map_err(|e| {
731                RemoteError::new_ex(
732                    RemoteErrorType::IoError,
733                    format!("Could not write file: {}", e),
734                )
735            })?;
736            n += bytes.len() as u64;
737        }
738
739        Ok(n)
740    }
741}
742
743#[cfg(test)]
744mod test {
745
746    #[cfg(feature = "with-s3-ci")]
747    use std::env;
748    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
749    use std::io::Cursor;
750    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
751    use std::time::SystemTime;
752
753    use pretty_assertions::assert_eq;
754
755    use super::*;
756    use crate::mock::container::Minio;
757
758    #[test]
759    fn should_init_s3() {
760        let s3 = AwsS3Fs::new("aws-s3-test", &Arc::new(Runtime::new().unwrap()));
761        assert_eq!(s3.wrkdir.as_path(), Path::new("/"));
762        assert_eq!(s3.bucket_name.as_str(), "aws-s3-test");
763        assert!(s3.region.is_none());
764        assert!(s3.endpoint.is_none());
765        assert_eq!(s3.is_anonymous(), true);
766        assert_eq!(s3.new_path_style, false);
767        assert!(s3.client.is_none());
768        assert!(s3.access_key.is_none());
769        assert!(s3.profile.is_none());
770        assert!(s3.secret_key.is_none());
771        assert!(s3.security_token.is_none());
772        assert!(s3.session_token.is_none());
773        assert!(s3.secret_key.is_none());
774        assert!(s3.client.is_none());
775    }
776
777    #[test]
778    fn should_init_s3_with_options() {
779        let s3 = AwsS3Fs::new("aws-s3-test", &Arc::new(Runtime::new().unwrap()))
780            .region("eu-central-1")
781            .access_key("AKIA0000")
782            .profile("default")
783            .secret_access_key("PASSWORD")
784            .security_token("secret")
785            .session_token("token")
786            .new_path_style(true)
787            .endpoint("omar");
788        assert_eq!(s3.bucket_name.as_str(), "aws-s3-test");
789        assert_eq!(s3.region.as_deref().unwrap(), "eu-central-1");
790        assert_eq!(s3.access_key.as_deref().unwrap(), "AKIA0000");
791        assert_eq!(s3.secret_key.as_deref().unwrap(), "PASSWORD");
792        assert_eq!(s3.security_token.as_deref().unwrap(), "secret");
793        assert_eq!(s3.session_token.as_deref().unwrap(), "token");
794        assert_eq!(s3.endpoint.as_deref().unwrap(), "omar");
795        assert_eq!(s3.is_anonymous(), false);
796        assert_eq!(s3.new_path_style, true);
797    }
798
799    #[test]
800    fn s3_is_direct_child() {
801        assert_eq!(AwsS3Fs::is_direct_child("pippo/", ""), true);
802        assert_eq!(AwsS3Fs::is_direct_child("pippo/sottocartella/", ""), false);
803        assert_eq!(
804            AwsS3Fs::is_direct_child("pippo/sottocartella/", "pippo/"),
805            true
806        );
807        assert_eq!(
808            AwsS3Fs::is_direct_child("pippo/sottocartella/", "pippo"), // This case must be handled indeed
809            false
810        );
811        assert_eq!(
812            AwsS3Fs::is_direct_child("pippo/sottocartella/readme.md", "pippo/sottocartella/"),
813            true
814        );
815        assert_eq!(
816            AwsS3Fs::is_direct_child("pippo/sottocartella/readme.md", "pippo/sottocartella/"),
817            true
818        );
819    }
820
821    #[test]
822    fn s3_resolve() {
823        let mut s3 = AwsS3Fs::new("aws-s3-test", &Arc::new(Runtime::new().unwrap()));
824        s3.wrkdir = PathBuf::from("/tmp");
825        // Absolute
826        assert_eq!(
827            s3.resolve(Path::new("/tmp/sottocartella/")).as_path(),
828            Path::new("tmp/sottocartella")
829        );
830        // Relative
831        assert_eq!(
832            s3.resolve(Path::new("subfolder/")).as_path(),
833            Path::new("tmp/subfolder")
834        );
835    }
836
837    #[test]
838    fn s3_fmt_path() {
839        assert_eq!(
840            AwsS3Fs::fmt_path(Path::new("/tmp/omar.txt"), false).as_str(),
841            "tmp/omar.txt"
842        );
843        assert_eq!(
844            AwsS3Fs::fmt_path(Path::new("omar.txt"), false).as_str(),
845            "omar.txt"
846        );
847        assert_eq!(
848            AwsS3Fs::fmt_path(Path::new("/tmp/subfolder"), true).as_str(),
849            "tmp/subfolder/"
850        );
851        assert_eq!(
852            AwsS3Fs::fmt_path(Path::new("tmp/subfolder"), true).as_str(),
853            "tmp/subfolder/"
854        );
855        assert_eq!(AwsS3Fs::fmt_path(Path::new("tmp"), true).as_str(), "tmp/");
856        assert_eq!(AwsS3Fs::fmt_path(Path::new("tmp/"), true).as_str(), "tmp/");
857        assert_eq!(AwsS3Fs::fmt_path(Path::new("/"), true).as_str(), "");
858    }
859
860    #[test]
861    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
862    fn should_not_append_to_file() {
863        crate::mock::logger();
864        let Ctx {
865            mut client,
866            container: _container,
867        } = setup_client();
868        // Create file
869        let p = Path::new("a.txt");
870        // Append to file
871        let file_data = "Hello, world!\n";
872        let reader = Cursor::new(file_data.as_bytes());
873        assert!(
874            client
875                .append_file(p, &Metadata::default(), Box::new(reader))
876                .is_err()
877        );
878        finalize_client(client);
879    }
880
881    #[test]
882    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
883    fn should_change_directory() {
884        crate::mock::logger();
885        let Ctx {
886            mut client,
887            container: _container,
888        } = setup_client();
889        let pwd = client.pwd().ok().unwrap();
890        assert!(client.change_dir(Path::new("/")).is_ok());
891        assert!(client.change_dir(pwd.as_path()).is_ok());
892        finalize_client(client);
893    }
894
895    #[test]
896    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
897    fn should_not_change_directory() {
898        crate::mock::logger();
899        let Ctx {
900            mut client,
901            container: _container,
902        } = setup_client();
903        assert!(
904            client
905                .change_dir(Path::new("/tmp/sdfghjuireghiuergh/useghiyuwegh"))
906                .is_err()
907        );
908        finalize_client(client);
909    }
910
911    #[test]
912    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
913    fn should_not_copy_file() {
914        crate::mock::logger();
915        let Ctx {
916            mut client,
917            container: _container,
918        } = setup_client();
919        // Create file
920        let p = Path::new("a.txt");
921        let file_data = "test data\n";
922        let reader = Cursor::new(file_data.as_bytes());
923        let mut metadata = Metadata::default();
924        metadata.size = file_data.len() as u64;
925        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
926        assert!(client.copy(p, Path::new("aaa/bbbb/ccc/b.txt")).is_err());
927        finalize_client(client);
928    }
929
930    #[test]
931    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
932    fn should_create_directory() {
933        crate::mock::logger();
934        let Ctx {
935            mut client,
936            container: _container,
937        } = setup_client();
938        // create directory
939        assert!(
940            client
941                .create_dir(Path::new("mydir"), UnixPex::from(0o755))
942                .is_ok()
943        );
944        finalize_client(client);
945    }
946
947    #[test]
948    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
949    fn should_not_create_directory_cause_already_exists() {
950        crate::mock::logger();
951        let Ctx {
952            mut client,
953            container: _container,
954        } = setup_client();
955        // create directory
956        assert!(
957            client
958                .create_dir(Path::new("mydir"), UnixPex::from(0o755))
959                .is_ok()
960        );
961        assert_eq!(
962            client
963                .create_dir(Path::new("mydir"), UnixPex::from(0o755))
964                .err()
965                .unwrap()
966                .kind,
967            RemoteErrorType::DirectoryAlreadyExists
968        );
969        finalize_client(client);
970    }
971
972    #[test]
973    #[cfg(feature = "with-s3-ci")]
974    fn should_not_create_directory() {
975        crate::mock::logger();
976        let Ctx {
977            mut client,
978            container: _container,
979        } = setup_client();
980        // create directory
981        assert!(
982            client
983                .create_dir(
984                    Path::new("/tmp/werfgjwerughjwurih/iwerjghiwgui"),
985                    UnixPex::from(0o755)
986                )
987                .is_err()
988        );
989        finalize_client(client);
990    }
991
992    #[test]
993    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
994    fn should_create_file() {
995        crate::mock::logger();
996        let Ctx {
997            mut client,
998            container: _container,
999        } = setup_client();
1000        // Create file
1001        let p = Path::new("a.txt");
1002        let file_data = "test data\n";
1003        let reader = Cursor::new(file_data.as_bytes());
1004        let mut metadata = Metadata::default();
1005        metadata.size = file_data.len() as u64;
1006        assert_eq!(
1007            client
1008                .create_file(p, &metadata, Box::new(reader))
1009                .ok()
1010                .unwrap(),
1011            10
1012        );
1013        // Verify size
1014        assert_eq!(client.stat(p).ok().unwrap().metadata().size, 10);
1015        finalize_client(client);
1016    }
1017
1018    #[test]
1019    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1020    fn should_not_exec_command() {
1021        crate::mock::logger();
1022        let Ctx {
1023            mut client,
1024            container: _container,
1025        } = setup_client();
1026        assert!(client.exec("echo 5").is_err());
1027        finalize_client(client);
1028    }
1029
1030    #[test]
1031    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1032    fn should_tell_whether_file_exists() {
1033        crate::mock::logger();
1034        let Ctx {
1035            mut client,
1036            container: _container,
1037        } = setup_client();
1038        // Create file
1039        let p = Path::new("a.txt");
1040        let file_data = "test data\n";
1041        let reader = Cursor::new(file_data.as_bytes());
1042        let mut metadata = Metadata::default();
1043        metadata.size = file_data.len() as u64;
1044        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
1045        // Verify size
1046        assert_eq!(client.exists(p).ok().unwrap(), true);
1047        assert_eq!(client.exists(Path::new("b.txt")).ok().unwrap(), false);
1048        assert_eq!(
1049            client.exists(Path::new("/tmp/ppppp/bhhrhu")).ok().unwrap(),
1050            false
1051        );
1052        finalize_client(client);
1053    }
1054
1055    #[test]
1056    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1057    fn should_list_dir() {
1058        crate::mock::logger();
1059        let Ctx {
1060            mut client,
1061            container: _container,
1062        } = setup_client();
1063        // Create file
1064        let wrkdir = client.pwd().ok().unwrap();
1065        let p = Path::new("a.txt");
1066        let file_data = "test data\n";
1067        let reader = Cursor::new(file_data.as_bytes());
1068        let mut metadata = Metadata::default();
1069        metadata.size = file_data.len() as u64;
1070        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
1071        // Verify size
1072        let file = client
1073            .list_dir(wrkdir.as_path())
1074            .ok()
1075            .unwrap()
1076            .get(0)
1077            .unwrap()
1078            .clone();
1079        assert_eq!(file.name().as_str(), "a.txt");
1080        let mut expected_path = wrkdir;
1081        expected_path.push(p);
1082        assert_eq!(file.path.as_path(), expected_path.as_path());
1083        assert_eq!(file.extension().as_deref().unwrap(), "txt");
1084        assert_eq!(file.metadata.size, 10);
1085        assert_eq!(file.metadata.mode, None);
1086        finalize_client(client);
1087    }
1088
1089    #[test]
1090    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1091    fn should_not_move_file() {
1092        crate::mock::logger();
1093        let Ctx {
1094            mut client,
1095            container: _container,
1096        } = setup_client();
1097        // Create file
1098        let p = Path::new("a.txt");
1099        let file_data = "test data\n";
1100        let reader = Cursor::new(file_data.as_bytes());
1101        let mut metadata = Metadata::default();
1102        metadata.size = file_data.len() as u64;
1103        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
1104        let dest = Path::new("b.txt");
1105        assert!(client.mov(p, dest).is_err());
1106        finalize_client(client);
1107    }
1108
1109    #[test]
1110    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1111    fn should_open_file() {
1112        crate::mock::logger();
1113        let Ctx {
1114            mut client,
1115            container: _container,
1116        } = setup_client();
1117        // Create file
1118        let p = Path::new("a.txt");
1119        let file_data = "test data\n";
1120        let reader = Cursor::new(file_data.as_bytes());
1121        let mut metadata = Metadata::default();
1122        metadata.size = file_data.len() as u64;
1123        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
1124        // Verify size
1125        let buffer: Box<dyn std::io::Write + Send> = Box::new(Vec::with_capacity(512));
1126        assert_eq!(client.open_file(p, buffer).ok().unwrap(), 10);
1127        finalize_client(client);
1128    }
1129
1130    #[test]
1131    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1132    fn should_write_big_data() {
1133        crate::mock::logger();
1134        let Ctx {
1135            mut client,
1136            container: _container,
1137        } = setup_client();
1138        // Create file
1139        let p = Path::new("a.txt");
1140        let file_data = vec![1; MIN_MULTIPART_UPLOAD_SIZE * 2];
1141        let reader = Cursor::new(file_data);
1142        let mut metadata = Metadata::default();
1143        metadata.size = (MIN_MULTIPART_UPLOAD_SIZE * 2) as u64;
1144        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
1145        // Verify size
1146        let buffer: Box<dyn std::io::Write + Send> = Box::new(Vec::with_capacity(512));
1147        assert_eq!(
1148            client.open_file(p, buffer).ok().unwrap(),
1149            (MIN_MULTIPART_UPLOAD_SIZE * 2) as u64
1150        );
1151        finalize_client(client);
1152    }
1153
1154    #[test]
1155    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1156    fn should_not_open_file() {
1157        crate::mock::logger();
1158        let Ctx {
1159            mut client,
1160            container: _container,
1161        } = setup_client();
1162        // Verify size
1163        let buffer: Box<dyn std::io::Write + Send> = Box::new(Vec::with_capacity(512));
1164        assert!(
1165            client
1166                .open_file(Path::new("/tmp/aashafb/hhh"), buffer)
1167                .is_err()
1168        );
1169        finalize_client(client);
1170    }
1171
1172    #[test]
1173    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1174    fn should_print_working_directory() {
1175        crate::mock::logger();
1176        let Ctx {
1177            mut client,
1178            container: _container,
1179        } = setup_client();
1180        assert!(client.pwd().is_ok());
1181        finalize_client(client);
1182    }
1183
1184    #[test]
1185    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1186    fn should_remove_dir_all() {
1187        crate::mock::logger();
1188        let Ctx {
1189            mut client,
1190            container: _container,
1191        } = setup_client();
1192        // Create dir
1193        let mut dir_path = client.pwd().ok().unwrap();
1194        dir_path.push(Path::new("test/"));
1195        assert!(
1196            client
1197                .create_dir(dir_path.as_path(), UnixPex::from(0o775))
1198                .is_ok()
1199        );
1200        // Create file
1201        let mut file_path = dir_path.clone();
1202        file_path.push(Path::new("a.txt"));
1203        let file_data = "test data\n";
1204        let reader = Cursor::new(file_data.as_bytes());
1205        let mut metadata = Metadata::default();
1206        metadata.size = file_data.len() as u64;
1207        assert!(
1208            client
1209                .create_file(file_path.as_path(), &metadata, Box::new(reader))
1210                .is_ok()
1211        );
1212        // Remove dir
1213        assert!(client.remove_dir_all(dir_path.as_path()).is_ok());
1214        finalize_client(client);
1215    }
1216
1217    #[test]
1218    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1219    fn should_remove_dir() {
1220        crate::mock::logger();
1221        let Ctx {
1222            mut client,
1223            container: _container,
1224        } = setup_client();
1225        // Create dir
1226        let mut dir_path = client.pwd().ok().unwrap();
1227        dir_path.push(Path::new("test/"));
1228        assert!(
1229            client
1230                .create_dir(dir_path.as_path(), UnixPex::from(0o775))
1231                .is_ok()
1232        );
1233        assert!(client.remove_dir(dir_path.as_path()).is_ok());
1234        finalize_client(client);
1235    }
1236
1237    #[test]
1238    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1239    fn should_not_remove_dir() {
1240        crate::mock::logger();
1241        let Ctx {
1242            mut client,
1243            container: _container,
1244        } = setup_client();
1245        // Remove dir
1246        assert!(client.remove_dir(Path::new("test/")).is_err());
1247        finalize_client(client);
1248    }
1249
1250    #[test]
1251    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1252    fn should_remove_file() {
1253        crate::mock::logger();
1254        let Ctx {
1255            mut client,
1256            container: _container,
1257        } = setup_client();
1258        // Create file
1259        let p = Path::new("a.txt");
1260        let file_data = "test data\n";
1261        let reader = Cursor::new(file_data.as_bytes());
1262        let mut metadata = Metadata::default();
1263        metadata.size = file_data.len() as u64;
1264        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
1265        assert!(client.remove_file(p).is_ok());
1266        // stat
1267        assert!(client.stat(p).is_err());
1268        finalize_client(client);
1269    }
1270
1271    #[test]
1272    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1273    fn should_not_setstat_file() {
1274        crate::mock::logger();
1275        let Ctx {
1276            mut client,
1277            container: _container,
1278        } = setup_client();
1279        // Create file
1280        let p = Path::new("a.sh");
1281        let file_data = "echo 5\n";
1282        let reader = Cursor::new(file_data.as_bytes());
1283        let mut metadata = Metadata::default();
1284        metadata.size = file_data.len() as u64;
1285        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
1286        assert!(
1287            client
1288                .setstat(
1289                    p,
1290                    Metadata {
1291                        accessed: Some(SystemTime::UNIX_EPOCH),
1292                        created: Some(SystemTime::UNIX_EPOCH),
1293                        gid: Some(1000),
1294                        file_type: remotefs::fs::FileType::File,
1295                        mode: Some(UnixPex::from(0o755)),
1296                        modified: Some(SystemTime::UNIX_EPOCH),
1297                        size: 7,
1298                        symlink: None,
1299                        uid: Some(1000),
1300                    }
1301                )
1302                .is_err()
1303        );
1304        finalize_client(client);
1305    }
1306
1307    #[test]
1308    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1309    fn should_stat_file() {
1310        crate::mock::logger();
1311        let Ctx {
1312            mut client,
1313            container: _container,
1314        } = setup_client();
1315        // Create file
1316        let p = Path::new("a.sh");
1317        let file_data = "echo 5\n";
1318        let reader = Cursor::new(file_data.as_bytes());
1319        let mut metadata = Metadata::default();
1320        metadata.size = file_data.len() as u64;
1321        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
1322        let entry = client.stat(p).ok().unwrap();
1323        assert_eq!(entry.name(), "a.sh");
1324        let mut expected_path = client.pwd().ok().unwrap();
1325        expected_path.push("a.sh");
1326        assert_eq!(entry.path(), expected_path.as_path());
1327        let meta = entry.metadata();
1328        assert_eq!(meta.size, 7);
1329        finalize_client(client);
1330    }
1331
1332    #[test]
1333    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1334    fn should_not_stat_file() {
1335        crate::mock::logger();
1336        let Ctx {
1337            mut client,
1338            container: _container,
1339        } = setup_client();
1340        // Create file
1341        let p = Path::new("a.sh");
1342        assert!(client.stat(p).is_err());
1343        finalize_client(client);
1344    }
1345
1346    #[test]
1347    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1348    fn should_make_symlink() {
1349        crate::mock::logger();
1350        let Ctx {
1351            mut client,
1352            container: _container,
1353        } = setup_client();
1354        // Create file
1355        let p = Path::new("a.sh");
1356        let file_data = "echo 5\n";
1357        let reader = Cursor::new(file_data.as_bytes());
1358        let mut metadata = Metadata::default();
1359        metadata.size = file_data.len() as u64;
1360        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
1361        let symlink = Path::new("b.sh");
1362        assert!(client.symlink(symlink, p).is_err());
1363        finalize_client(client);
1364    }
1365
1366    #[test]
1367    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1368    fn should_not_make_symlink() {
1369        crate::mock::logger();
1370        let Ctx {
1371            mut client,
1372            container: _container,
1373        } = setup_client();
1374        // Create file
1375        let p = Path::new("a.sh");
1376        let file_data = "echo 5\n";
1377        let reader = Cursor::new(file_data.as_bytes());
1378        let mut metadata = Metadata::default();
1379        metadata.size = file_data.len() as u64;
1380        assert!(client.create_file(p, &metadata, Box::new(reader)).is_ok());
1381        let symlink = Path::new("b.sh");
1382        let file_data = "echo 5\n";
1383        let reader = Cursor::new(file_data.as_bytes());
1384        assert!(
1385            client
1386                .create_file(symlink, &metadata, Box::new(reader))
1387                .is_ok()
1388        );
1389        assert!(client.symlink(symlink, p).is_err());
1390        assert!(client.remove_file(symlink).is_ok());
1391        assert!(client.symlink(symlink, Path::new("c.sh")).is_err());
1392        finalize_client(client);
1393    }
1394
1395    #[test]
1396    fn should_return_errors_on_uninitialized_client() {
1397        let mut client =
1398            AwsS3Fs::new("aws-s3-test", &Arc::new(Runtime::new().unwrap())).region("eu-central-1");
1399        assert!(client.change_dir(Path::new("/tmp")).is_err());
1400        assert!(
1401            client
1402                .copy(Path::new("/nowhere"), PathBuf::from("/culonia").as_path())
1403                .is_err()
1404        );
1405        assert!(client.exec("echo 5").is_err());
1406        assert!(client.disconnect().is_err());
1407        assert!(client.symlink(Path::new("/a"), Path::new("/b")).is_err());
1408        assert!(client.list_dir(Path::new("/tmp")).is_err());
1409        assert!(
1410            client
1411                .create_dir(Path::new("/tmp"), UnixPex::from(0o755))
1412                .is_err()
1413        );
1414        assert!(client.pwd().is_err());
1415        assert!(client.remove_dir_all(Path::new("/nowhere")).is_err());
1416        assert!(
1417            client
1418                .mov(Path::new("/nowhere"), Path::new("/culonia"))
1419                .is_err()
1420        );
1421        assert!(client.stat(Path::new("/tmp")).is_err());
1422        assert!(
1423            client
1424                .setstat(Path::new("/tmp"), Metadata::default())
1425                .is_err()
1426        );
1427        assert!(client.open(Path::new("/tmp/pippo.txt")).is_err());
1428        assert!(
1429            client
1430                .create(Path::new("/tmp/pippo.txt"), &Metadata::default())
1431                .is_err()
1432        );
1433        assert!(
1434            client
1435                .append(Path::new("/tmp/pippo.txt"), &Metadata::default())
1436                .is_err()
1437        );
1438    }
1439
1440    fn is_send<T: Send>(_send: T) {}
1441
1442    fn is_sync<T: Sync>(_sync: T) {}
1443
1444    #[test]
1445    fn test_should_be_sync() {
1446        let client = AwsS3Fs::new("bucket", &Arc::new(Runtime::new().unwrap()));
1447
1448        is_sync(client);
1449    }
1450
1451    #[test]
1452    fn test_should_be_send() {
1453        let client = AwsS3Fs::new("bucket", &Arc::new(Runtime::new().unwrap()));
1454
1455        is_send(client);
1456    }
1457
1458    // -- test utils
1459
1460    #[allow(dead_code)]
1461    struct Ctx {
1462        client: AwsS3Fs,
1463        #[cfg(feature = "with-containers")]
1464        container: Minio,
1465        #[cfg(all(feature = "with-s3-ci", not(feature = "with-containers")))]
1466        container: (),
1467    }
1468
1469    #[cfg(all(feature = "with-s3-ci", not(feature = "with-containers")))]
1470    fn setup_client() -> Ctx {
1471        // Get transfer
1472        let bucket = env!("AWS_S3_BUCKET");
1473        let mut client = AwsS3Fs::new(bucket, &Arc::new(Runtime::new().unwrap()));
1474        assert!(client.connect().is_ok());
1475        // Create wrkdir
1476        let tempdir = PathBuf::from(generate_tempdir());
1477        assert!(
1478            client
1479                .create_dir(tempdir.as_path(), UnixPex::from(0o775))
1480                .is_ok()
1481        );
1482        // Change directory
1483        let err = client.change_dir(tempdir.as_path());
1484        if err.is_err() {
1485            println!("Error: {:?}", err);
1486        }
1487        assert!(client.change_dir(tempdir.as_path()).is_ok());
1488        Ctx {
1489            client,
1490            container: (),
1491        }
1492    }
1493
1494    #[cfg(feature = "with-containers")]
1495    fn setup_client() -> Ctx {
1496        let minio = Minio::start();
1497        let port = minio.port();
1498
1499        // Get transfer
1500        let runtime = Arc::new(Runtime::new().expect("Could not create runtime"));
1501        let mut client = AwsS3Fs::new("github-ci", &runtime)
1502            .endpoint(format!("http://localhost:{port}"))
1503            .access_key("minioadmin")
1504            .secret_access_key("minioadmin")
1505            .new_path_style(true);
1506
1507        // connect
1508        assert!(client.connect().is_ok());
1509
1510        // Create bucket manually
1511        let fut = client
1512            .client()
1513            .unwrap()
1514            .create_bucket()
1515            .bucket("github-ci")
1516            .send();
1517        let res = runtime.block_on(fut);
1518
1519        assert!(res.is_ok());
1520
1521        // Create wrkdir
1522        let tempdir = PathBuf::from(generate_tempdir());
1523        assert!(
1524            client
1525                .create_dir(tempdir.as_path(), UnixPex::from(0o775))
1526                .is_ok()
1527        );
1528        // Change directory
1529        assert!(client.change_dir(tempdir.as_path()).is_ok());
1530        Ctx {
1531            client,
1532            container: minio,
1533        }
1534    }
1535
1536    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1537    fn finalize_client(mut client: AwsS3Fs) {
1538        // Get working directory
1539        let wrkdir = client.pwd().ok().unwrap();
1540        // Remove directory
1541        assert!(client.remove_dir_all(wrkdir.as_path()).is_ok());
1542        assert!(client.disconnect().is_ok());
1543    }
1544
1545    #[cfg(any(feature = "with-s3-ci", feature = "with-containers"))]
1546    fn generate_tempdir() -> String {
1547        use rand::distr::Alphanumeric;
1548        use rand::{Rng, rng};
1549        let mut rng = rng();
1550        let name: String = std::iter::repeat(())
1551            .map(|()| rng.sample(Alphanumeric))
1552            .map(char::from)
1553            .take(8)
1554            .collect();
1555        format!("/github-ci/temp_{}/", name)
1556    }
1557}
remotefs_aws_s3/client.rs

remotefs_aws_s3/
client.rs
