quilt-rs 0.31.1

Rust library for accessing Quilt data packages.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

use std::path::Path;

use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ChecksumAlgorithm;
use aws_sdk_s3::types::CompletedMultipartUpload;
use aws_sdk_s3::types::CompletedPart;
use aws_smithy_types::byte_stream::Length;

use crate::Res;
use crate::checksum::ObjectHash;
use crate::checksum::Sha256ChunkedHash;
use crate::checksum::chunksize_and_parts;
use crate::error::ChecksumError;
use crate::error::S3Error;
use crate::error::S3ErrorKind;
use crate::io::remote::describe_sdk_error;
use quilt_uri::S3Uri;

pub async fn multipart_upload_and_sha256_chunksum(
    client: aws_sdk_s3::Client,
    source_path: impl AsRef<Path>,
    dest_uri: &S3Uri,
    size: u64,
) -> Res<(S3Uri, ObjectHash)> {
    let (chunksize, num_chunks) = chunksize_and_parts(size);
    let upload_id = client
        .create_multipart_upload()
        .bucket(&dest_uri.bucket)
        .key(&dest_uri.key)
        .checksum_algorithm(ChecksumAlgorithm::Sha256)
        .send()
        .await
        .map_err(|err| S3Error::new(S3ErrorKind::Raw(describe_sdk_error(err))))?
        .upload_id
        .ok_or(S3Error::new(S3ErrorKind::UploadId(
            "failed to get an UploadId".to_string(),
        )))?;

    let mut parts: Vec<CompletedPart> = Vec::new();
    for chunk_idx in 0..num_chunks {
        let part_number = chunk_idx as i32 + 1;
        let offset = chunk_idx * chunksize;
        let length = chunksize.min(size - offset);
        let chunk_body = ByteStream::read_from()
            .path(source_path.as_ref())
            .offset(offset)
            .length(Length::Exact(length)) // https://github.com/awslabs/aws-sdk-rust/issues/821
            .build()
            .await?;
        let part_response = client
            .upload_part()
            .bucket(&dest_uri.bucket)
            .key(&dest_uri.key)
            .upload_id(&upload_id)
            .part_number(part_number)
            .checksum_algorithm(ChecksumAlgorithm::Sha256)
            .body(chunk_body)
            .send()
            .await
            .map_err(|err| S3Error::new(S3ErrorKind::Raw(describe_sdk_error(err))))?;
        parts.push(
            CompletedPart::builder()
                .part_number(part_number)
                .e_tag(part_response.e_tag.unwrap_or_default())
                .checksum_sha256(part_response.checksum_sha256.unwrap_or_default())
                .build(),
        );
    }

    let response = client
        .complete_multipart_upload()
        .bucket(&dest_uri.bucket)
        .key(&dest_uri.key)
        .upload_id(&upload_id)
        .multipart_upload(
            CompletedMultipartUpload::builder()
                .set_parts(Some(parts))
                .build(),
        )
        .send()
        .await
        .map_err(|err| S3Error::new(S3ErrorKind::Raw(describe_sdk_error(err))))?;

    let s3_checksum = response
        .checksum_sha256
        .ok_or(ChecksumError::NoS3Checksum("missing checksum".to_string()))?;
    let (checksum_b64, _) =
        s3_checksum
            .split_once('-')
            .ok_or(ChecksumError::Malformed(format!(
                "expected 'sha256-parts' format, got: {s3_checksum}"
            )))?;

    Ok((
        S3Uri {
            version: response.version_id,
            ..dest_uri.clone()
        },
        Sha256ChunkedHash::try_from(checksum_b64)?.into(),
    ))
}