s3-algo 0.7.4

High-performance algorithms for batch operations to Amazon S3
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

use clap::{Arg, Command};
use s3_algo::*;
use std::io::Write;
use std::path::{Path, PathBuf};

#[tokio::main]
async fn main() {
    let matches = Command::new("perf_data")
        .about("Upload a directory to S3 on localhost.")
        .arg(
            Arg::new("source")
                .help("Path to a folder to upload to S3")
                .required(true)
                .index(1),
        )
        .arg(
            Arg::new("dest_bucket")
                .help("Destination bucket")
                .required(true)
                .index(2),
        )
        .arg(
            Arg::new("dest_prefix")
                .help("Destination prefix")
                .required(true)
                .index(3),
        )
        .arg(
            Arg::new("parallelization")
                .short('n')
                .long("parallelization")
                .value_name("N")
                .help("Maximum number of simultaneous upload requests"),
        )
        .get_matches();

    let path = matches.get_one::<String>("source").unwrap();
    let bucket = matches.get_one::<String>("dest_bucket").unwrap();
    let prefix = matches.get_one::<String>("dest_prefix").unwrap();
    let parallelization = matches
        .get_one::<String>("parallelization")
        .and_then(|s| s.parse::<usize>().ok())
        .unwrap_or(10);

    benchmark_s3_upload(
        Path::new(path).to_path_buf(),
        bucket.to_owned(),
        prefix.to_owned(),
        parallelization,
    )
    .await;
    println!("Done");
}

async fn benchmark_s3_upload(
    dir_path: PathBuf,
    bucket: String,
    prefix: String,
    copy_parallelization: usize,
) {
    let cfg = Config {
        copy_parallelization,
        ..Default::default()
    };
    let s3 = testing_sdk_client().await;
    let algo = S3Algo::with_config(s3, cfg);

    upload_perf_log_init(&mut std::io::stdout());
    let progress = |res| async move { upload_perf_log_update(&mut std::io::stdout(), res) };

    algo.upload_files(
        bucket,
        files_recursive(dir_path, PathBuf::from(&prefix)),
        progress,
        |client| client.put_object(),
    )
    .await
    .unwrap();
}

// Helpers for writing data
macro_rules! write_cell {
    ($out:expr, $x:expr) => {
        let _ = write!($out, "{0: >18}", format!("{:.5}", $x));
    };
}
pub fn upload_perf_log_init<W: Write>(out: &mut W) {
    let _ = writeln!(
        out,
        "{0: >w$}{1: >w$}{2: >w$}{3: >w$}{4: >w$}{5: >w$}",
        "attempts",
        "bytes",
        "success_ms",
        "total_ms",
        "MBps",
        "MBps est",
        w = 18
    );
}
pub fn upload_perf_log_update<W: Write>(out: &mut W, res: RequestReport) {
    // TODO: Write performance data to file with tokio
    let megabytes = res.size as f64 / 1_000_000.0;
    let speed = megabytes / res.success_time.as_secs_f64();
    write_cell!(out, res.attempts);
    write_cell!(out, res.size);
    write_cell!(out, res.success_time.as_millis());
    write_cell!(out, res.total_time.as_millis());
    write_cell!(out, speed);
    write_cell!(out, res.est);
    let _ = writeln!(out);
}