esthri 11.0.2

Extremely simple (memory stable) S3 client that supports get, put, head, list, and sync.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

/*
 * Copyright (C) 2022 Swift Navigation Inc.
 * Contact: Swift Navigation <dev@swiftnav.com>
 *
 * This source is subject to the license found in the file 'LICENSE' which must
 * be be distributed together with this source. All other rights reserved.
 *
 * THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
 * EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
 */

use std::{path::Path, time::Duration};

use async_compression::tokio::write::GzipDecoder;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::presigning::PresigningConfig;
use aws_sdk_s3::Client as S3Client;
use futures::TryStreamExt;
use reqwest::Client as HttpClient;
use tokio::{
    fs::File,
    io::{copy, AsyncWriteExt},
};
use tokio_util::io::StreamReader;

use crate::{opts::EsthriGetOptParams, Error, Result};

use super::DEAFULT_EXPIRATION;

/// Generate a presigned URL for a client to use to download a file.
/// The file can be downloaded using an HTTP GET on this URL.
pub async fn presign_get(
    s3: &S3Client,
    bucket: impl AsRef<str>,
    key: impl AsRef<str>,
    expiration: Option<Duration>,
) -> Result<String> {
    let presigning_config = PresigningConfig::builder()
        .expires_in(expiration.unwrap_or(DEAFULT_EXPIRATION))
        .build()
        .map_err(Error::PresigningConfigError)?;

    let presigned_req = s3
        .get_object()
        .bucket(bucket.as_ref().to_string())
        .key(key.as_ref().to_string())
        .presigned(presigning_config)
        .await
        .map_err(|e| match e {
            SdkError::ServiceError(error) => Error::GetObjectFailed(Box::new(error.into_err())),
            _ => Error::SdkError(e.to_string()),
        })?;

    Ok(presigned_req.uri().to_string())
}

/// Helper to download a file using a presigned URL, taking care of transparent
/// compression.
pub async fn download_file_presigned(
    client: &HttpClient,
    presigned_url: &str,
    filepath: &Path,
    opts: &EsthriGetOptParams,
) -> Result<()> {
    let mut file = File::create(filepath).await?;
    let mut resp = client.get(presigned_url).send().await?.error_for_status()?;
    if opts.transparent_compression {
        let st = resp
            .bytes_stream()
            .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e));
        let mut src = StreamReader::new(st);
        let mut dest = GzipDecoder::new(file);
        copy(&mut src, &mut dest).await?;
    } else {
        while let Some(bytes) = resp.chunk().await? {
            file.write_all(&bytes).await?;
            file.flush().await?;
        }
    }
    Ok(())
}