mod auth;
mod config;
mod region;
pub use auth::{Mac, PutPolicy};
pub use config::QiniuConfig;
pub use region::Region;
use std::collections::BTreeMap;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
use bytes::Bytes;
use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE};
use reqwest::multipart::{Form, Part};
use serde::Deserialize;
use crate::error::{Error, Result};
use crate::storage::{ListOptions, ObjectStorage};
use crate::types::{ListOutcome, ObjectMetadata, PutBody, PutOptions};
use auth::urlsafe_b64;
const KEY_SET: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'~')
.remove(b'/');
const QUERY_VALUE_SET: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'~');
const QINIU_NO_SUCH_FILE: u16 = 612;
const DEFAULT_TOKEN_TTL: u64 = 3600;
const DEFAULT_PART_SIZE: usize = 4 * 1024 * 1024;
const MIN_PART_SIZE: usize = 1024 * 1024;
const MAX_PARTS: u32 = 10_000;
const PART_RETRY: usize = 3;
#[derive(Debug, Clone)]
pub struct QiniuClient {
config: QiniuConfig,
mac: Mac,
http: reqwest::Client,
}
impl QiniuClient {
pub fn new(config: QiniuConfig) -> Self {
QiniuClient {
mac: Mac::new(&config.access_key, &config.secret_key),
http: reqwest::Client::new(),
config,
}
}
pub fn with_http(mut self, http: reqwest::Client) -> Self {
self.http = http;
self
}
pub fn config(&self) -> &QiniuConfig {
&self.config
}
pub fn upload_token(&self, key: &str, ttl: Duration) -> Result<String> {
let deadline = unix_now() + ttl.as_secs() as i64;
let policy = PutPolicy::for_key(&self.config.bucket, key, deadline);
self.mac.upload_token(&policy)
}
fn make_upload_token(&self, key: &str, opts: &PutOptions) -> Result<String> {
let deadline = unix_now() + DEFAULT_TOKEN_TTL as i64;
let policy =
PutPolicy::for_key(&self.config.bucket, key, deadline).insert_only(opts.insert_only);
self.mac.upload_token(&policy)
}
fn encode_key(&self, key: &str) -> String {
utf8_percent_encode(key, KEY_SET).to_string()
}
fn encoded_entry(&self, key: &str) -> String {
let entry = format!("{}:{}", self.config.bucket, key);
urlsafe_b64(entry.as_bytes())
}
fn download_url(&self, key: &str, ttl: Duration) -> String {
let encoded_path = format!("/{}", self.encode_key(key));
let base = format!(
"{}://{}{}",
self.config.scheme(),
self.config.download_host(),
encoded_path
);
if !self.config.private_bucket {
return base;
}
let deadline = unix_now() + ttl.as_secs() as i64;
self.mac
.private_download_url(&base, &encoded_path, deadline)
}
async fn check(&self, resp: reqwest::Response) -> Result<reqwest::Response> {
let status = resp.status();
if status.is_success() {
return Ok(resp);
}
let body = resp.text().await.unwrap_or_default();
let short: String = body.chars().take(512).collect();
Err(Error::api(
status.as_u16(),
Some(status.as_u16() as i64),
short,
))
}
pub async fn initiate_multipart_upload(
&self,
key: &str,
opts: &PutOptions,
) -> Result<MultipartUpload> {
let token = self.make_upload_token(key, opts)?;
let enc_bucket = urlsafe_b64(self.config.bucket.as_bytes());
let enc_key = urlsafe_b64(key.as_bytes());
let path = format!("/buckets/{enc_bucket}/objects/{enc_key}/uploads");
let url = format!(
"{}://{}{path}",
self.config.scheme(),
self.config.up_host_resolved()
);
let resp = self
.http
.post(&url)
.header(
reqwest::header::AUTHORIZATION,
Mac::up_authorization(&token),
)
.header(CONTENT_TYPE, "text/plain;charset=utf8")
.send()
.await?;
let resp = self.check(resp).await?;
#[derive(Deserialize)]
struct InitResp {
#[serde(rename = "uploadId")]
upload_id: String,
}
let init: InitResp = resp.json().await?;
Ok(MultipartUpload {
client: self.clone(),
upload_id: init.upload_id,
key: key.to_string(),
token,
})
}
pub fn multipart_upload(
&self,
key: &str,
upload_id: impl Into<String>,
) -> Result<MultipartUpload> {
let token = self.make_upload_token(key, &PutOptions::default())?;
Ok(MultipartUpload {
client: self.clone(),
upload_id: upload_id.into(),
key: key.to_string(),
token,
})
}
async fn upload_resumable(
&self,
key: &str,
data: Bytes,
part_size: usize,
opts: &PutOptions,
resume: Option<&ResumeState>,
) -> Result<ObjectMetadata> {
if data.is_empty() {
return self.put(key, PutBody::Bytes(data), opts).await;
}
let part_size = if part_size == 0 {
DEFAULT_PART_SIZE
} else {
part_size.max(MIN_PART_SIZE)
};
let total = data.len();
let total_parts = (total + part_size - 1) / part_size;
if total_parts > MAX_PARTS as usize {
return Err(Error::InvalidInput(format!(
"need {total_parts} parts (> {MAX_PARTS}); use a larger part_size"
)));
}
let (mp, mut done) = match resume {
Some(state) => (
self.multipart_upload(key, state.upload_id.clone())?,
state.parts.clone(),
),
None => (
self.initiate_multipart_upload(key, opts).await?,
BTreeMap::new(),
),
};
let mut parts: Vec<CompletedPart> = Vec::with_capacity(total_parts);
for idx in 0..total_parts {
let number = (idx + 1) as u32;
if let Some(name) = done.get(&number) {
parts.push(CompletedPart {
number,
name: name.clone(),
});
continue;
}
let start = idx * part_size;
let end = (start + part_size).min(total);
let chunk = data.slice(start..end);
let name = mp.upload_part_with_retry(number, chunk).await?;
done.insert(number, name.clone());
parts.push(CompletedPart { number, name });
}
mp.complete(&parts).await
}
pub async fn put_resumable_resume(
&self,
key: &str,
body: PutBody,
part_size: usize,
opts: &PutOptions,
state: &ResumeState,
) -> Result<ObjectMetadata> {
self.upload_resumable(key, body.into_bytes(), part_size, opts, Some(state))
.await
}
}
fn unix_now() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
#[async_trait]
impl ObjectStorage for QiniuClient {
fn backend(&self) -> &'static str {
"qiniu"
}
fn bucket(&self) -> &str {
&self.config.bucket
}
async fn put(&self, key: &str, body: PutBody, opts: &PutOptions) -> Result<ObjectMetadata> {
let data = body.into_bytes();
let size = data.len() as u64;
let token = self.make_upload_token(key, opts)?;
let mut part =
Part::stream(data).file_name(key.rsplit('/').next().unwrap_or(key).to_string());
if let Some(ct) = &opts.content_type {
if let Ok(value) = HeaderValue::from_str(ct) {
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, value);
part = part.headers(headers);
}
}
let form = Form::new()
.text("token", token)
.text("key", key.to_string())
.part("file", part);
let url = format!(
"{}://{}/",
self.config.scheme(),
self.config.up_host_resolved()
);
let resp = self.http.post(&url).multipart(form).send().await?;
let resp = self.check(resp).await?;
#[derive(Deserialize)]
struct UpResp {
#[serde(default)]
hash: Option<String>,
}
let up: UpResp = resp.json().await?;
Ok(ObjectMetadata {
key: key.to_string(),
size,
content_type: opts.content_type.clone(),
etag: up.hash,
last_modified: None,
})
}
async fn put_resumable(
&self,
key: &str,
body: PutBody,
part_size: usize,
opts: &PutOptions,
) -> Result<ObjectMetadata> {
self.upload_resumable(key, body.into_bytes(), part_size, opts, None)
.await
}
async fn get(&self, key: &str) -> Result<Bytes> {
let url = self.download_url(key, Duration::from_secs(DEFAULT_TOKEN_TTL));
let resp = self.check(self.http.get(&url).send().await?).await?;
Ok(resp.bytes().await?)
}
async fn delete(&self, key: &str) -> Result<()> {
let path = format!("/delete/{}", self.encoded_entry(key));
let url = format!(
"{}://{}{}",
self.config.scheme(),
self.config.rs_host_resolved(),
path
);
let auth = self.mac.qbox_authorization(&format!("{path}\n"));
let resp = self
.http
.post(&url)
.header(reqwest::header::AUTHORIZATION, auth)
.send()
.await?;
self.check(resp).await?;
Ok(())
}
async fn exists(&self, key: &str) -> Result<bool> {
match self.stat(key).await {
Ok(_) => Ok(true),
Err(Error::Api { status, .. }) if status == QINIU_NO_SUCH_FILE => Ok(false),
Err(other) => Err(other),
}
}
async fn stat(&self, key: &str) -> Result<ObjectMetadata> {
let path = format!("/stat/{}", self.encoded_entry(key));
let url = format!(
"{}://{}{}",
self.config.scheme(),
self.config.rs_host_resolved(),
path
);
let auth = self.mac.qbox_authorization(&format!("{path}\n"));
let resp = self
.http
.post(&url)
.header(reqwest::header::AUTHORIZATION, auth)
.send()
.await?;
let resp = self.check(resp).await?;
let stat: StatResp = resp.json().await?;
Ok(ObjectMetadata {
key: key.to_string(),
size: stat.fsize.unwrap_or(0),
content_type: stat.mime_type,
etag: stat.hash,
last_modified: stat.put_time.map(qiniu_puttime_to_system),
})
}
async fn list(&self, prefix: &str, opts: &ListOptions) -> Result<ListOutcome> {
let mut pairs: Vec<String> = Vec::new();
pairs.push(format!(
"bucket={}",
utf8_percent_encode(&self.config.bucket, QUERY_VALUE_SET)
));
if !prefix.is_empty() {
pairs.push(format!(
"prefix={}",
utf8_percent_encode(prefix, QUERY_VALUE_SET)
));
}
if let Some(delim) = &opts.delimiter {
pairs.push(format!(
"delimiter={}",
utf8_percent_encode(delim, QUERY_VALUE_SET)
));
}
if let Some(marker) = &opts.marker {
pairs.push(format!(
"marker={}",
utf8_percent_encode(marker, QUERY_VALUE_SET)
));
}
pairs.push(format!("limit={}", opts.limit.unwrap_or(100)));
let query = pairs.join("&");
let path = "/list";
let url = format!(
"{}://{}{}?{}",
self.config.scheme(),
self.config.rsf_host_resolved(),
path,
query
);
let auth = self.mac.qbox_authorization(&format!("{path}?{query}\n"));
let resp = self
.http
.get(&url)
.header(reqwest::header::AUTHORIZATION, auth)
.send()
.await?;
let resp = self.check(resp).await?;
let page: ListResp = resp.json().await?;
let items = page
.items
.into_iter()
.map(|i| ObjectMetadata {
key: i.key,
size: i.fsize.unwrap_or(0),
content_type: i.mime_type,
etag: i.hash,
last_modified: i.put_time.map(qiniu_puttime_to_system),
})
.collect();
Ok(ListOutcome {
items,
next_marker: page.marker.filter(|m| !m.is_empty()),
})
}
fn presign_get(&self, key: &str, expires: Duration) -> Result<String> {
Ok(self.download_url(key, expires))
}
}
#[derive(Deserialize)]
struct StatResp {
#[serde(default)]
fsize: Option<u64>,
#[serde(default)]
hash: Option<String>,
#[serde(default, rename = "mimeType")]
mime_type: Option<String>,
#[serde(default, rename = "putTime")]
put_time: Option<i64>,
}
#[derive(Deserialize)]
struct ListResp {
#[serde(default)]
items: Vec<ListItem>,
#[serde(default)]
marker: Option<String>,
}
#[derive(Deserialize)]
struct ListItem {
key: String,
#[serde(default)]
fsize: Option<u64>,
#[serde(default)]
hash: Option<String>,
#[serde(default, rename = "mimeType")]
mime_type: Option<String>,
#[serde(default, rename = "putTime")]
put_time: Option<i64>,
}
fn qiniu_puttime_to_system(put_time: i64) -> SystemTime {
let secs = (put_time / 10_000_000).max(0) as u64;
UNIX_EPOCH + Duration::from_secs(secs)
}
#[derive(Debug, Clone)]
pub struct MultipartUpload {
client: QiniuClient,
upload_id: String,
key: String,
token: String,
}
impl MultipartUpload {
pub fn upload_id(&self) -> &str {
&self.upload_id
}
pub fn key(&self) -> &str {
&self.key
}
pub async fn upload_part(&self, part_number: u32, data: Bytes) -> Result<String> {
if part_number == 0 || part_number > MAX_PARTS {
return Err(Error::InvalidInput(
"part_number must be within 1..=10000".into(),
));
}
let url = format!(
"{}://{}/uploadIds/{}/{}",
self.client.config.scheme(),
self.client.config.up_host_resolved(),
self.upload_id,
part_number
);
let resp = self
.client
.http
.put(&url)
.header(
reqwest::header::AUTHORIZATION,
Mac::up_authorization(&self.token),
)
.header(CONTENT_TYPE, "application/octet-stream")
.body(data)
.send()
.await?;
let resp = self.client.check(resp).await?;
#[derive(Deserialize)]
struct PartResp {
#[serde(rename = "partName")]
part_name: String,
}
let pr: PartResp = resp.json().await?;
Ok(pr.part_name)
}
async fn upload_part_with_retry(&self, part_number: u32, data: Bytes) -> Result<String> {
let mut last: Option<Error> = None;
for _ in 0..PART_RETRY {
match self.upload_part(part_number, data.clone()).await {
Ok(name) => return Ok(name),
Err(e) => last = Some(e),
}
}
Err(last.unwrap_or_else(|| Error::InvalidInput("upload_part exhausted retries".into())))
}
pub async fn complete(&self, parts: &[CompletedPart]) -> Result<ObjectMetadata> {
if parts.is_empty() {
return Err(Error::InvalidInput("no parts to complete".into()));
}
let joined = parts
.iter()
.map(|p| p.name.as_str())
.collect::<Vec<_>>()
.join(",");
let etaets = urlsafe_b64(joined.as_bytes());
let query = format!("op=mkfile&etaets={etaets}&etaets_cnt={}", parts.len());
let url = format!(
"{}://{}/uploadIds/{}?{}",
self.client.config.scheme(),
self.client.config.up_host_resolved(),
self.upload_id,
query
);
let resp = self
.client
.http
.post(&url)
.header(
reqwest::header::AUTHORIZATION,
Mac::up_authorization(&self.token),
)
.header(CONTENT_TYPE, "text/plain;charset=utf8")
.send()
.await?;
let resp = self.client.check(resp).await?;
#[derive(Deserialize)]
struct CompleteResp {
#[serde(default)]
key: Option<String>,
#[serde(default)]
hash: Option<String>,
#[serde(default, rename = "mimeType")]
mime_type: Option<String>,
#[serde(default)]
fsize: Option<u64>,
}
let c: CompleteResp = resp.json().await?;
Ok(ObjectMetadata {
key: c.key.unwrap_or_else(|| self.key.clone()),
size: c.fsize.unwrap_or(0),
content_type: c.mime_type,
etag: c.hash,
last_modified: None,
})
}
}
#[derive(Debug, Clone)]
pub struct CompletedPart {
pub number: u32,
pub name: String,
}
#[derive(Debug, Clone)]
pub struct ResumeState {
pub upload_id: String,
pub parts: BTreeMap<u32, String>,
}