biolib 1.3.117

BioLib client library and CLI for running applications on BioLib
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

use std::collections::HashMap;

use crate::api::ApiClient;

const MIN_CHUNK_SIZE: usize = 10_000_000; // 10 MB
const MAX_CHUNK_COUNT: usize = 9_000;

pub fn upload_module_input(
    api_client: &mut ApiClient,
    job_uuid: &str,
    job_auth_token: &str,
    module_input: &[u8],
) -> crate::Result<()> {
    let mut headers = HashMap::new();
    headers.insert("job-auth-token".to_string(), job_auth_token.to_string());

    let start_response = api_client.start_multipart_upload(
        &format!("/jobs/{job_uuid}/storage/input/start_upload/"),
        Some(&headers),
    )?;

    let chunk_size = calculate_chunk_size(module_input.len());
    let mut parts = Vec::new();
    let mut offset = 0usize;
    let mut part_number = 1u32;

    while offset < module_input.len() {
        let end = std::cmp::min(offset + chunk_size, module_input.len());
        let chunk = &module_input[offset..end];

        let presigned_url = api_client.get_presigned_upload_url(
            &format!("/jobs/{job_uuid}/storage/input/presigned_upload_url/"),
            part_number,
            &start_response.upload_id,
            Some(&headers),
        )?;

        let etag = api_client.upload_to_presigned_url(&presigned_url, chunk)?;

        parts.push(serde_json::json!({
            "ETag": etag,
            "PartNumber": part_number,
        }));

        offset = end;
        part_number += 1;
    }

    api_client.complete_multipart_upload(
        &format!("/jobs/{job_uuid}/storage/input/complete_upload/"),
        &start_response.upload_id,
        &parts,
        Some(&headers),
    )?;

    Ok(())
}

fn calculate_chunk_size(total_size: usize) -> usize {
    std::cmp::max(MIN_CHUNK_SIZE, total_size / MAX_CHUNK_COUNT)
}