1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
//! object操作相关
use crate::acl;
use crate::client;
pub use crate::request::{
    CompleteMultipartUpload, ErrNo, InitiateMultipartUploadResult, Part, Request, Response,
};
pub use mime;
pub use quick_xml::de::from_str;
pub use quick_xml::se::to_string;
pub use reqwest::Body;
use std::collections::HashMap;
use std::fs;
use std::io::Cursor;

#[async_trait::async_trait]
pub trait Objects {
    /// 上传本地小文件
    async fn put_object(
        &self,
        content_type: mime::Mime,
        key: &str,
        data: Vec<u8>,
        acl_header: Option<&acl::AclHeader>,
    ) -> Response;

    /// 上传本地大文件
    async fn put_big_object(
        &self,
        file_path: &str,
        key: &str,
        content_type: mime::Mime,
        storage_class: &str,
        acl_header: Option<&acl::AclHeader>,
        part_size: u64,
    ) -> Response;

    /// 判断文件是否存在
    async fn head_object(&self, key: &str) -> Response;

    /// 删除文件
    async fn delete_object(&self, key: &str) -> Response;

    /// 获取文件二进制流
    async fn get_object_binary(&self, key: &str) -> Response;

    /// 下载文件到本地
    async fn get_object(&self, key: &str, file_name: &str) -> Response;

    /// 获取分块上传的upload_id
    async fn put_object_get_upload_id(
        &self,
        key: &str,
        content_type: &mime::Mime,
        storage_class: &str,
        acl_header: Option<&acl::AclHeader>,
    ) -> Response;

    /// 分块上传
    async fn put_object_part(
        &self,
        key: &str,
        upload_id: &str,
        part_number: u64,
        body: Vec<u8>,
        content_type: &mime::Mime,
        acl_header: Option<&acl::AclHeader>,
    ) -> Response;

    /// 完成分块上传
    async fn put_object_complete_part(
        &self,
        key: &str,
        etag_map: &HashMap<u64, String>,
        upload_id: &str,
    ) -> Response;

    /// Abort Multipart Upload 用来实现舍弃一个分块上传并删除已上传的块。
    /// 当您调用 Abort Multipart Upload 时,如果有正在使用这个 Upload Parts 上传块的请求,
    /// 则 Upload Parts 会返回失败。当该 UploadId 不存在时,会返回404 NoSuchUpload。
    async fn abort_object_part(&self, key: &str, upload_id: &str) -> Response;
}

#[async_trait::async_trait]
impl Objects for client::Client {
    /// 上传本地小文件
    /// 见[官网文档](https://cloud.tencent.com/document/product/436/7749)
    /// # Examples
    /// ```
    /// use rust_qcos::client::Client;
    /// use rust_qcos::objects::Objects;
    /// use mime;
    /// use rust_qcos::acl::{AclHeader, ObjectAcl};
    /// async {
    /// let mut acl_header = AclHeader::new();
    /// acl_header.insert_object_x_cos_acl(ObjectAcl::AuthenticatedRead);
    /// let client = Client::new("foo", "bar", "qcloudtest-1256650966", "ap-guangzhou");
    /// let data = std::fs::read("Cargo.toml").unwrap();
    /// let res = client.put_object(mime::TEXT_PLAIN_UTF_8, "Cargo.toml", data, None).await;
    /// assert!(res.error_message.contains("403"));
    /// };
    /// ```
    async fn put_object(
        &self,
        content_type: mime::Mime,
        key: &str,
        data: Vec<u8>,
        acl_header: Option<&acl::AclHeader>,
    ) -> Response {
        let mut headers = self.gen_common_headers();
        headers.insert("Content-Type".to_string(), content_type.to_string());
        headers.insert("Content-Length".to_string(), data.len().to_string());
        let url_path = self.get_path_from_object_key(key);
        headers =
            self.get_headers_with_auth("put", url_path.as_str(), acl_header, Some(headers), None);
        let resp = Request::put(
            self.get_full_url_from_path(url_path.as_str()).as_str(),
            None,
            Some(&headers),
            None,
            None,
            Some(reqwest::Body::from(data)),
        )
        .await;
        self.make_response(resp)
    }

    /// 上传本地大文件
    /// 见[官网文档](https://cloud.tencent.com/document/product/436/7749)
    /// # Examples
    /// ```
    /// use rust_qcos::client::Client;
    /// use rust_qcos::objects::Objects;
    /// use mime;
    /// use rust_qcos::acl::{AclHeader, ObjectAcl};
    /// async {
    /// let mut acl_header = AclHeader::new();
    /// acl_header.insert_object_x_cos_acl(ObjectAcl::AuthenticatedRead);
    /// let client = Client::new("foo", "bar", "qcloudtest-1256650966", "ap-guangzhou");
    /// // 分块传输
    /// let res = client.put_big_object("Cargo.toml","Cargo.toml", mime::TEXT_PLAIN_UTF_8, "ARCHIVE", Some(&acl_header), 1024 * 1024 * 100).await;
    /// assert!(res.error_message.contains("403"));
    /// };
    /// ```
    async fn put_big_object(
        &self,
        file_path: &str,
        key: &str,
        content_type: mime::Mime,
        storage_class: &str,
        acl_header: Option<&acl::AclHeader>,
        part_size: u64,
    ) -> Response {
        use tokio::io::AsyncReadExt;
        use tokio::io::AsyncSeekExt;
        use tokio::io::SeekFrom;
        assert!(part_size > 0);
        let mut file = match tokio::fs::File::open(file_path).await {
            Ok(file) => file,
            Err(e) => {
                return Response::new(
                    ErrNo::IO,
                    format!("打开文件失败: {}, {}", file_path, e),
                    Default::default(),
                )
            }
        };
        // 设置为分块上传或者大于5G会启动分块上传
        let file_size = match file.metadata().await {
            Ok(meta) => meta.len(),
            Err(e) => {
                return Response::new(
                    ErrNo::IO,
                    format!("获取文件大小失败: {}, {}", file_path, e),
                    Default::default(),
                )
            }
        };
        let mut part_number = 1;
        let mut start: u64;
        let mut etag_map = HashMap::new();
        let upload_id = self
            .put_object_get_upload_id(key, &content_type, storage_class, acl_header)
            .await;
        if upload_id.error_no != ErrNo::SUCCESS {
            return upload_id;
        }
        let upload_id = String::from_utf8_lossy(&upload_id.result[..]).to_string();
        loop {
            start = part_size * (part_number - 1);
            if start >= file_size {
                // 调用合并
                let resp = self
                    .put_object_complete_part(key, &etag_map, upload_id.as_str())
                    .await;
                if resp.error_no != ErrNo::SUCCESS {
                    // 调用清理
                    self.abort_object_part(key, upload_id.as_str()).await;
                }
                return resp;
            }
            let mut size = part_size;
            // 计算剩余的大小
            if file_size - start < part_size {
                size = file_size - start;
            }
            // 如果剩余的块小于1M, 那么要全部上传
            if file_size - size - start <= 1024 * 1024 {
                size = file_size - start;
            }
            if let Err(e) = file.seek(SeekFrom::Start(start)).await {
                // 调用清理
                self.abort_object_part(key, upload_id.as_str()).await;
                return Response::new(
                    ErrNo::IO,
                    format!("设置文件指针失败: {}, {}", file_path, e),
                    Default::default(),
                );
            }
            let mut body: Vec<u8> = vec![0; size as usize];
            if let Err(e) = file.read_exact(&mut body).await {
                // 调用清理
                self.abort_object_part(key, upload_id.as_str()).await;
                return Response::new(
                    ErrNo::IO,
                    format!("读取文件失败: {}, {}", file_path, e),
                    Default::default(),
                );
            }
            let resp = self
                .put_object_part(
                    key,
                    upload_id.as_str(),
                    part_number,
                    body,
                    &content_type,
                    acl_header,
                )
                .await;
            if resp.error_no != ErrNo::SUCCESS {
                // 调用清理
                self.abort_object_part(key, upload_id.as_str()).await;
                return resp;
            }
            etag_map.insert(part_number, resp.headers["etag"].clone());
            part_number += 1;
        }
    }

    async fn head_object(&self, key: &str) -> Response {
        let url_path = self.get_path_from_object_key(key);
        let headers = self.get_headers_with_auth("head", url_path.as_str(), None, None, None);
        let resp = Request::head(
            self.get_full_url_from_path(url_path.as_str()).as_str(),
            None,
            Some(&headers),
        )
        .await;
        match resp {
            Ok(e) => e,
            Err(e) => e,
        }
    }

    /// 删除文件
    /// 见[官网文档](https://cloud.tencent.com/document/product/436/7743)
    /// # Examples
    /// ```
    /// use rust_qcos::client::Client;
    /// use rust_qcos::objects::Objects;
    /// async {
    /// let client = Client::new("foo", "bar", "qcloudtest-1256650966", "ap-guangzhou");
    /// let res = client.delete_object("Cargo.toml").await;
    /// assert!(res.error_message.contains("403"))
    /// };
    /// ```
    async fn delete_object(&self, key: &str) -> Response {
        let url_path = self.get_path_from_object_key(key);
        let headers = self.get_headers_with_auth("delete", url_path.as_str(), None, None, None);
        let resp = Request::delete(
            self.get_full_url_from_path(url_path.as_str()).as_str(),
            None,
            Some(&headers),
            None,
            None,
        )
        .await;
        match resp {
            Ok(e) => e,
            Err(e) => e,
        }
    }

    /// 下载文件二进制流
    /// 见[官网文档](https://cloud.tencent.com/document/product/436/7753)
    /// # Examples
    /// ```
    /// use rust_qcos::client::Client;
    /// use rust_qcos::objects::Objects;
    /// async {
    /// let client = Client::new("foo", "bar", "qcloudtest-1256650966", "ap-guangzhou");
    /// let res = client.get_object_binary("Cargo.toml").await;
    /// assert!(res.error_message.contains("403"));
    /// };
    /// ```
    async fn get_object_binary(&self, key: &str) -> Response {
        let url_path = self.get_path_from_object_key(key);
        let headers = self.get_headers_with_auth("get", url_path.as_str(), None, None, None);
        let resp = Request::get(
            self.get_full_url_from_path(url_path.as_str()).as_str(),
            None,
            Some(&headers),
        )
        .await;
        self.make_response(resp)
    }

    /// 下载文件到本地
    /// 见[官网文档](https://cloud.tencent.com/document/product/436/7753)
    /// # Examples
    /// ```
    /// use rust_qcos::client::Client;
    /// use rust_qcos::objects::Objects;
    /// async {
    /// let client = Client::new("foo", "bar", "qcloudtest-1256650966", "ap-guangzhou");
    /// let res = client.get_object("Cargo.toml", "Cargo.toml").await;
    /// assert!(res.error_message.contains("403"));
    /// };
    /// ```
    async fn get_object(&self, key: &str, file_name: &str) -> Response {
        let resp = self.get_object_binary(key).await;
        if resp.error_no == ErrNo::SUCCESS {
            let output_file_r = fs::File::create(file_name);
            let mut output_file;
            match output_file_r {
                Ok(e) => output_file = e,
                Err(e) => {
                    return Response::new(
                        ErrNo::OTHER,
                        format!("创建文件失败: {}", e),
                        "".to_string(),
                    );
                }
            }
            if let Err(e) = std::io::copy(&mut Cursor::new(resp.result), &mut output_file) {
                return Response::new(ErrNo::OTHER, format!("下载文件失败: {}", e), "".to_string());
            }
            return Response::blank_success();
        }
        resp
    }
    /// 请求实现初始化分块上传,成功执行此请求后将返回 UploadId,用于后续的 Upload Part 请求
    /// [官网文档](https://cloud.tencent.com/document/product/436/7746)
    async fn put_object_get_upload_id(
        &self,
        key: &str,
        content_type: &mime::Mime,
        storage_class: &str,
        acl_header: Option<&acl::AclHeader>,
    ) -> Response {
        let mut query = HashMap::new();
        query.insert("uploads".to_string(), "".to_string());
        let url_path = self.get_path_from_object_key(key);
        let mut headers = self.gen_common_headers();
        headers.insert("Content-Type".to_string(), content_type.to_string());
        headers.insert("x-cos-storage-class".to_string(), storage_class.to_string());
        let headers = self.get_headers_with_auth(
            "post",
            url_path.as_str(),
            acl_header,
            Some(headers),
            Some(&query),
        );
        let resp = Request::post(
            self.get_full_url_from_path(url_path.as_str()).as_str(),
            Some(&query),
            Some(&headers),
            None,
            None,
            None as Option<Body>,
        )
        .await;
        match resp {
            Ok(res) => {
                if res.error_no != ErrNo::SUCCESS {
                    return res;
                }
                match quick_xml::de::from_slice::<InitiateMultipartUploadResult>(&res.result[..]) {
                    Ok(res) => Response::new(ErrNo::SUCCESS, "".to_string(), res.upload_id),
                    Err(e) => Response::new(ErrNo::DECODE, e.to_string(), Default::default()),
                }
            }
            Err(e) => e,
        }
    }

    /// 分块上传文件
    /// [官网文档](https://cloud.tencent.com/document/product/436/7750)
    async fn put_object_part(
        &self,
        key: &str,
        upload_id: &str,
        part_number: u64,
        body: Vec<u8>,
        content_type: &mime::Mime,
        acl_header: Option<&acl::AclHeader>,
    ) -> Response {
        let mut headers = self.gen_common_headers();
        headers.insert("Content-Type".to_string(), content_type.to_string());
        headers.insert("Content-Length".to_string(), body.len().to_string());
        let url_path = self.get_path_from_object_key(key);
        let mut query = HashMap::new();
        query.insert("partNumber".to_string(), part_number.to_string());
        query.insert("uploadId".to_string(), upload_id.to_string());
        headers = self.get_headers_with_auth(
            "put",
            url_path.as_str(),
            acl_header,
            Some(headers),
            Some(&query),
        );
        let resp = Request::put(
            self.get_full_url_from_path(url_path.as_str()).as_str(),
            Some(&query),
            Some(&headers),
            None,
            None,
            Some(body),
        )
        .await;
        self.make_response(resp)
    }

    /// 完成分块上传
    /// [官网文档](https://cloud.tencent.com/document/product/436/7742)
    async fn put_object_complete_part(
        &self,
        key: &str,
        etag_map: &HashMap<u64, String>,
        upload_id: &str,
    ) -> Response {
        let url_path = self.get_path_from_object_key(key);
        let mut query = HashMap::new();
        query.insert("uploadId".to_string(), upload_id.to_string());
        let mut headers = self.gen_common_headers();
        headers.insert("Content-Type".to_string(), "application/xml".to_string());
        let headers = self.get_headers_with_auth(
            "post",
            url_path.as_str(),
            None,
            Some(headers),
            Some(&query),
        );
        let mut parts = Vec::new();
        // 按part_number排序
        let mut keys = Vec::new();
        for k in etag_map.keys() {
            keys.push(k);
        }
        keys.sort();
        for k in keys {
            parts.push(Part {
                part_number: *k,
                etag: etag_map[&k].clone(),
            })
        }
        let complete = CompleteMultipartUpload { part: parts };
        let serialized_str = match to_string(&complete) {
            Ok(s) => s,
            Err(e) => return Response::new(ErrNo::ENCODE, e.to_string(), Default::default()),
        };
        let resp = Request::post(
            self.get_full_url_from_path(url_path.as_str()).as_str(),
            Some(&query),
            Some(&headers),
            None,
            None,
            Some(serialized_str),
        )
        .await;
        self.make_response(resp)
    }

    /// 终止分块上传,清理文件碎片
    /// [官网文档](https://cloud.tencent.com/document/product/436/7740)
    async fn abort_object_part(&self, key: &str, upload_id: &str) -> Response {
        let url_path = self.get_path_from_object_key(key);
        let mut query = HashMap::new();
        query.insert("uploadId".to_string(), upload_id.to_string());
        let headers =
            self.get_headers_with_auth("delete", url_path.as_str(), None, None, Some(&query));
        let resp = Request::delete(
            self.get_full_url_from_path(url_path.as_str()).as_str(),
            Some(&query),
            Some(&headers),
            None,
            None,
        )
        .await;
        self.make_response(resp)
    }
}