use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use fake_server::{FakeServer, Received, Response};
use origincloud_core::{
AccessToken, CONSISTENCY_TOKEN_HEADER, CallOptions, Client, ConsistencyTokenCell,
ConsistencyTokenStore, Error, Method, Operation, Request, Service, longrunning,
};
use serde_json::{Value, json};
const UNAVAILABLE: &str = r#"{"error":{"code":503,"status":"UNAVAILABLE","message":"the read model is behind","details":[
{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"SERVICE_UNAVAILABLE","domain":"origincloudapis.eu"},
{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"0.010s"}]}}"#;
const BAD_REQUEST: &str = r#"{"error":{"code":400,"status":"INVALID_ARGUMENT","message":"invalid input on field volume.sizeGib: must be at least 1","details":[
{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"BAD_REQUEST","domain":"origincloudapis.eu"},
{"@type":"type.googleapis.com/google.rpc.RequestInfo","requestId":"4bf92f3577b34da6"},
{"@type":"type.googleapis.com/google.rpc.BadRequest","fieldViolations":[{"field":"volume.sizeGib","description":"must be at least 1"}]}]}}"#;
const REPLAYED_CREATE: &str = r#"{"error":{"code":409,"status":"FAILED_PRECONDITION","message":"the service account organizations/o/serviceAccounts/s was created by an earlier request with this request_id; its client secret was returned only to that request. To get a working secret, delete that account and create a new one","details":[
{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"CONFLICT","domain":"origincloudapis.eu","metadata":{"serviceAccount":"organizations/o/serviceAccounts/s"}},
{"@type":"type.googleapis.com/google.rpc.RequestInfo","requestId":"4bf92f3577b34da6"}]}}"#;
const OPERATION_NAME: &str = "projects/p/zones/z/operations/op-1";
fn client(server: &FakeServer, max_attempts: u32) -> Client {
Client::builder()
.endpoint(server.url())
.credentials(AccessToken::new("test-token"))
.user_agent("my-tool/1.2")
.max_attempts(max_attempts)
.build()
.unwrap()
}
fn service(server: &FakeServer) -> Service {
client(server, 5).service("pvs", true)
}
async fn get(service: &Service, path: &str) -> Result<Value, Error> {
service
.call(Request::new(Method::GET, path).idempotent())
.await
}
#[tokio::test]
async fn a_request_carries_its_credentials_agent_query_and_body() {
let seen = Arc::new(Mutex::new(None::<Received>));
let recorded = Arc::clone(&seen);
let server = FakeServer::start(move |r| {
*recorded.lock().unwrap() = Some(r);
async { Response::json("{}") }
})
.await;
let _: Value = service(&server)
.call(
Request::new(Method::POST, "/v1beta1/projects/p/zones/z/volumes")
.with_query(vec![("pageSize".to_owned(), "10".to_owned())])
.with_body(br#"{"sizeGib":5}"#.to_vec()),
)
.await
.unwrap();
let got = seen.lock().unwrap().clone().unwrap();
assert_eq!(got.header("authorization"), Some("Bearer test-token"));
assert!(
got.header("user-agent")
.unwrap()
.starts_with("my-tool/1.2 origincloud-sdk-rust/"),
"{:?}",
got.header("user-agent")
);
assert_eq!(got.header("content-type"), Some("application/json"));
assert_eq!(got.text(), r#"{"sizeGib":5}"#);
assert_eq!(got.method, "POST");
assert_eq!(got.path, "/v1beta1/projects/p/zones/z/volumes");
assert_eq!(got.query, "pageSize=10");
}
#[derive(Debug)]
struct NoCredentials;
impl origincloud_core::credentials::Credentials for NoCredentials {
fn token(&self) -> origincloud_core::credentials::TokenFuture<'_> {
Box::pin(async { Err(Error::NoCredentials) })
}
}
#[tokio::test]
async fn an_unauthenticated_call_is_sent_without_credentials() {
let seen = Arc::new(Mutex::new(None::<Received>));
let recorded = Arc::clone(&seen);
let server = FakeServer::start(move |r| {
*recorded.lock().unwrap() = Some(r);
async { Response::json("{}") }
})
.await;
let client = Client::builder()
.endpoint(server.url())
.credentials(NoCredentials)
.build()
.unwrap();
let sts = client.service("sts", false);
let _: Value = sts
.call(Request::new(Method::POST, "/v1beta1:assumeServiceAccount").unauthenticated())
.await
.unwrap();
assert_eq!(
seen.lock()
.unwrap()
.clone()
.unwrap()
.header("authorization"),
None
);
let refused: Result<Value, Error> = sts
.call(Request::new(Method::POST, "/v1beta1:assumeServiceAccount"))
.await;
assert!(matches!(refused, Err(Error::NoCredentials)), "{refused:?}");
}
#[tokio::test]
async fn an_unauthenticated_call_carries_no_consistency_token() {
let seen = Arc::new(Mutex::new(Vec::<Received>::new()));
let recorded = Arc::clone(&seen);
let server = FakeServer::start(move |r| {
recorded.lock().unwrap().push(r);
async { Response::json("{}").with_header(CONSISTENCY_TOKEN_HEADER, "v1.anonymous") }
})
.await;
let client = client(&server, 1);
client.set_consistency_token("v1.held");
let billing = client.service("billing", false);
let _: Value = billing
.call(
Request::new(Method::GET, "/v1beta1/legalDocuments")
.idempotent()
.unauthenticated(),
)
.await
.unwrap();
assert_eq!(client.consistency_token(), "v1.held");
let _: Value = billing
.call(Request::new(Method::GET, "/v1beta1/legalDocuments").idempotent())
.await
.unwrap();
let seen = seen.lock().unwrap();
let anonymous = &seen[0];
for header in ["authorization", "origin-consistency-token", "content-type"] {
assert_eq!(anonymous.header(header), None, "{header}");
}
assert_eq!(anonymous.header("accept"), Some("application/json"));
assert!(
anonymous
.header("user-agent")
.unwrap()
.starts_with("my-tool/1.2 origincloud-sdk-rust/"),
"{:?}",
anonymous.header("user-agent")
);
let authenticated = &seen[1];
assert_eq!(
authenticated.header("authorization"),
Some("Bearer test-token")
);
assert!(authenticated.header("user-agent").is_some());
assert_eq!(
authenticated.header("origin-consistency-token"),
Some("v1.held")
);
}
#[tokio::test]
async fn an_idempotent_call_is_retried_while_the_answer_is_retryable() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |_| {
let n = counted.fetch_add(1, Ordering::SeqCst) + 1;
async move {
if n < 3 {
Response::json(UNAVAILABLE).with_status(503)
} else {
Response::json("{}")
}
}
})
.await;
get(&service(&server), "/x").await.unwrap();
assert_eq!(calls.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn a_call_that_isnt_idempotent_is_sent_once() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |_| {
counted.fetch_add(1, Ordering::SeqCst);
async { Response::json(UNAVAILABLE).with_status(503) }
})
.await;
let err = service(&server)
.call::<Value>(Request::new(Method::POST, "/x"))
.await
.unwrap_err();
assert_eq!(err.reason(), Some("SERVICE_UNAVAILABLE"));
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn retries_stop_at_the_most_attempts() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |_| {
counted.fetch_add(1, Ordering::SeqCst);
async { Response::json(UNAVAILABLE).with_status(503) }
})
.await;
let service = client(&server, 2).service("pvs", true);
let err = get(&service, "/x").await.unwrap_err();
assert!(err.is_retryable(), "{err}");
assert_eq!(calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn an_error_that_isnt_retryable_is_returned_at_once() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |_| {
counted.fetch_add(1, Ordering::SeqCst);
async { Response::json(BAD_REQUEST).with_status(400) }
})
.await;
let err = get(&service(&server), "/x").await.unwrap_err();
assert_eq!(calls.load(Ordering::SeqCst), 1);
let api = err.api().expect("an API error");
assert_eq!(api.http_status, Some(400));
assert_eq!(api.status, "INVALID_ARGUMENT");
assert_eq!(api.reason, "BAD_REQUEST");
assert_eq!(api.request_id, "4bf92f3577b34da6");
assert_eq!(api.field_violations.len(), 1);
assert_eq!(api.field_violations[0].field, "volume.sizeGib");
assert_eq!(
err.to_string(),
"origincloud: BAD_REQUEST: invalid input on field volume.sizeGib: must be at least 1 (request 4bf92f3577b34da6)"
);
}
#[tokio::test]
async fn a_conflict_is_returned_at_once_with_its_metadata() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |_| {
counted.fetch_add(1, Ordering::SeqCst);
async { Response::json(REPLAYED_CREATE).with_status(409) }
})
.await;
let err = service(&server)
.call::<Value>(Request::new(Method::POST, "/x").idempotent())
.await
.unwrap_err();
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert!(!err.is_retryable());
assert_eq!(err.reason(), Some("CONFLICT"));
let api = err.api().expect("an API error");
assert_eq!(api.http_status, Some(409));
assert_eq!(api.status, "FAILED_PRECONDITION");
assert_eq!(api.request_id, "4bf92f3577b34da6");
assert!(api.message.starts_with("the service account organizations/o/serviceAccounts/s was created by an earlier request"), "{}", api.message);
assert_eq!(
api.metadata.get("serviceAccount").map(String::as_str),
Some("organizations/o/serviceAccounts/s")
);
assert_eq!(api.metadata.len(), 1);
}
#[tokio::test]
async fn a_proxy_error_without_a_status_is_retried() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |_| {
let n = counted.fetch_add(1, Ordering::SeqCst) + 1;
async move {
if n == 1 {
Response::json("<html>bad gateway</html>").with_status(502)
} else {
Response::json("{}")
}
}
})
.await;
get(&service(&server), "/x").await.unwrap();
assert_eq!(calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn a_connection_that_fails_is_retried_for_an_idempotent_call() {
let server = FakeServer::start(|_| async { Response::json("{}") }).await;
let url = server.url();
drop(server);
tokio::time::sleep(Duration::from_millis(50)).await;
let client = Client::builder()
.endpoint(url)
.credentials(AccessToken::new("t"))
.max_attempts(2)
.build()
.unwrap();
let started = Instant::now();
let err = get(&client.service("pvs", true), "/x").await.unwrap_err();
assert!(matches!(err, Error::Transport(_)), "{err}");
assert!(
started.elapsed() >= Duration::from_millis(100),
"{:?}",
started.elapsed()
);
}
#[tokio::test]
async fn the_consistency_token_is_kept_and_sent_back() {
let presented = Arc::new(Mutex::new(Vec::<String>::new()));
let log = Arc::clone(&presented);
let server = FakeServer::start(move |r| {
log.lock().unwrap().push(
r.header(CONSISTENCY_TOKEN_HEADER)
.unwrap_or_default()
.to_owned(),
);
async move {
match r.path.as_str() {
"/mutate" => {
Response::json("{}").with_header(CONSISTENCY_TOKEN_HEADER, "v1.after-mutation")
}
"/fail" => Response::json(BAD_REQUEST)
.with_status(400)
.with_header(CONSISTENCY_TOKEN_HEADER, "v1.after-failure"),
_ => Response::json("{}"),
}
}
})
.await;
let client = client(&server, 1);
let service = client.service("pvs", true);
for path in ["/read", "/mutate", "/read", "/fail", "/read"] {
let _ = get(&service, path).await;
}
assert_eq!(
*presented.lock().unwrap(),
[
"",
"",
"v1.after-mutation",
"v1.after-mutation",
"v1.after-failure"
]
);
assert_eq!(client.consistency_token(), "v1.after-failure");
}
#[tokio::test]
async fn clients_given_one_store_share_the_token() {
let presented = Arc::new(Mutex::new(Vec::<String>::new()));
let log = Arc::clone(&presented);
let server = FakeServer::start(move |r| {
log.lock().unwrap().push(
r.header(CONSISTENCY_TOKEN_HEADER)
.unwrap_or_default()
.to_owned(),
);
async move {
match r.path.as_str() {
"/mutate" => {
Response::json("{}").with_header(CONSISTENCY_TOKEN_HEADER, "v1.written")
}
_ => Response::json("{}"),
}
}
})
.await;
let store = Arc::new(ConsistencyTokenCell::new());
let shared = |server: &FakeServer| {
Client::builder()
.endpoint(server.url())
.credentials(AccessToken::new("t"))
.consistency_token_store(store.clone())
.build()
.unwrap()
};
let (writer, reader) = (shared(&server), shared(&server));
let alone = client(&server, 1);
get(&writer.service("pvs", true), "/mutate").await.unwrap();
get(&reader.service("pvs", true), "/read").await.unwrap();
get(&alone.service("pvs", true), "/read").await.unwrap();
assert_eq!(*presented.lock().unwrap(), ["", "v1.written", ""]);
assert_eq!(store.get(), "v1.written");
assert_eq!(reader.consistency_token(), "v1.written");
reader.set_consistency_token("v1.set");
assert_eq!(writer.consistency_token(), "v1.set");
assert_eq!(alone.consistency_token(), "");
}
#[tokio::test]
async fn a_store_of_the_callers_own_holds_the_token() {
#[derive(Debug, Default)]
struct Logged {
token: Mutex<String>,
log: Mutex<Vec<String>>,
}
impl ConsistencyTokenStore for Logged {
fn get(&self) -> String {
let token = self.token.lock().unwrap().clone();
self.log.lock().unwrap().push(format!("get {token}"));
token
}
fn set(&self, token: String) {
self.log.lock().unwrap().push(format!("set {token}"));
*self.token.lock().unwrap() = token;
}
fn replace(&self, presented: &str, answered: String) {
self.log
.lock()
.unwrap()
.push(format!("replace {presented} with {answered}"));
*self.token.lock().unwrap() = answered;
}
}
let server = FakeServer::start(|_| async {
Response::json("{}").with_header(CONSISTENCY_TOKEN_HEADER, "v1.answered")
})
.await;
let store = Arc::new(Logged::default());
let client = Client::builder()
.endpoint(server.url())
.credentials(AccessToken::new("t"))
.consistency_token_store(store.clone())
.build()
.unwrap();
get(&client.service("pvs", true), "/x").await.unwrap();
get(&client.service("pvs", true), "/x").await.unwrap();
assert_eq!(
*store.log.lock().unwrap(),
[
"get ",
"replace with v1.answered",
"get v1.answered",
"replace v1.answered with v1.answered"
]
);
}
const MALFORMED_TOKEN: &str = r#"{"error":{"code":400,"status":"INVALID_ARGUMENT","message":"the Origin-Consistency-Token header must be a token a response of this API carried","details":[
{"@type":"type.googleapis.com/google.rpc.ErrorInfo","reason":"BAD_REQUEST","domain":"origincloudapis.eu"}]}}"#;
async fn refusing_tokens() -> (FakeServer, Arc<Mutex<Vec<String>>>) {
let presented = Arc::new(Mutex::new(Vec::<String>::new()));
let log = Arc::clone(&presented);
let server = FakeServer::start(move |r| {
let token = r
.header(CONSISTENCY_TOKEN_HEADER)
.unwrap_or_default()
.to_owned();
log.lock().unwrap().push(token.clone());
async move {
if token.is_empty() || token == "v1.good" {
Response::json("{}")
} else {
Response::json(MALFORMED_TOKEN).with_status(400)
}
}
})
.await;
(server, presented)
}
#[tokio::test]
async fn a_refused_token_is_dropped_and_the_call_sent_again() {
let (server, presented) = refusing_tokens().await;
let client = client(&server, 1);
client.set_consistency_token("v1.from-another-api");
let service = client.service("pvs", true);
let _: Value = service
.call(Request::new(Method::POST, "/mutate"))
.await
.unwrap();
assert_eq!(client.consistency_token(), "");
get(&service, "/read").await.unwrap();
assert_eq!(*presented.lock().unwrap(), ["v1.from-another-api", "", ""]);
}
#[tokio::test]
async fn a_refused_token_is_sent_again_once_only() {
#[derive(Debug)]
struct Stuck;
impl ConsistencyTokenStore for Stuck {
fn get(&self) -> String {
"v1.stuck".to_owned()
}
fn set(&self, _: String) {}
}
let (server, presented) = refusing_tokens().await;
let client = Client::builder()
.endpoint(server.url())
.credentials(AccessToken::new("t"))
.consistency_token_store(Arc::new(Stuck))
.build()
.unwrap();
let err = get(&client.service("pvs", true), "/read")
.await
.unwrap_err();
assert_eq!(err.api().unwrap().http_status, Some(400));
assert_eq!(*presented.lock().unwrap(), ["v1.stuck", "v1.stuck"]);
}
#[tokio::test]
async fn a_token_stored_while_the_refused_one_was_out_stays() {
let store = Arc::new(ConsistencyTokenCell::new());
store.set("v1.stale".to_owned());
let presented = Arc::new(Mutex::new(Vec::<String>::new()));
let (log, concurrent) = (Arc::clone(&presented), Arc::clone(&store));
let server = FakeServer::start(move |r| {
let token = r
.header(CONSISTENCY_TOKEN_HEADER)
.unwrap_or_default()
.to_owned();
log.lock().unwrap().push(token.clone());
if token == "v1.stale" {
concurrent.set("v1.newer".to_owned());
}
async move {
if token == "v1.stale" {
Response::json(MALFORMED_TOKEN).with_status(400)
} else {
Response::json("{}")
}
}
})
.await;
let client = Client::builder()
.endpoint(server.url())
.credentials(AccessToken::new("t"))
.consistency_token_store(store.clone())
.build()
.unwrap();
get(&client.service("pvs", true), "/read").await.unwrap();
assert_eq!(*presented.lock().unwrap(), ["v1.stale", "v1.newer"]);
assert_eq!(store.get(), "v1.newer");
}
#[tokio::test]
async fn another_bad_request_keeps_the_token() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |_| {
counted.fetch_add(1, Ordering::SeqCst);
async { Response::json(BAD_REQUEST).with_status(400) }
})
.await;
let client = client(&server, 5);
client.set_consistency_token("v1.held");
let err = get(&client.service("pvs", true), "/x").await.unwrap_err();
assert_eq!(err.reason(), Some("BAD_REQUEST"));
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(client.consistency_token(), "v1.held");
}
#[tokio::test]
async fn call_options_bound_the_attempts_of_one_service() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |_| {
counted.fetch_add(1, Ordering::SeqCst);
async { Response::json(UNAVAILABLE).with_status(503) }
})
.await;
let client = client(&server, 3);
let service = client.service("pvs", true);
let once = service.with_options(CallOptions::new().max_attempts(1));
assert_eq!(once.options().attempts(), Some(1));
assert_eq!(CallOptions::new().max_attempts(0).attempts(), Some(1));
let err = get(&once, "/x").await.unwrap_err();
assert!(err.is_retryable(), "{err}");
assert_eq!(calls.swap(0, Ordering::SeqCst), 1);
get(&service, "/x").await.unwrap_err();
assert_eq!(calls.swap(0, Ordering::SeqCst), 3);
let more = service.with_options(CallOptions::new().max_attempts(4));
get(&more, "/x").await.unwrap_err();
assert_eq!(calls.load(Ordering::SeqCst), 4);
}
#[tokio::test]
async fn waiting_follows_the_clients_settings_not_the_calls_options() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |_| {
let n = counted.fetch_add(1, Ordering::SeqCst) + 1;
async move {
if n == 1 {
Response::json(UNAVAILABLE).with_status(503)
} else {
Response::json(done_volume())
}
}
})
.await;
let once = client(&server, 3)
.service("pvs", true)
.with_options(CallOptions::new().max_attempts(1));
let volume = running(&once).wait().await.unwrap();
assert_eq!(volume["name"], "projects/p/zones/z/volumes/vol-1");
assert_eq!(calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn a_file_is_returned_as_it_was_sent_and_retried_like_a_read() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |r| {
let n = counted.fetch_add(1, Ordering::SeqCst) + 1;
async move {
assert_eq!(r.header("accept"), Some("*/*"));
if n < 2 {
Response::json(UNAVAILABLE).with_status(503)
} else {
Response {
status: 200,
headers: vec![("Content-Type".to_owned(), "image/png".to_owned())],
body: vec![0x89, b'P', b'N', b'G', 0, 0xff],
}
}
}
})
.await;
let file = service(&server)
.call_http_body(Request::new(Method::GET, "/x:download").idempotent())
.await
.unwrap();
assert_eq!(file.content_type, "image/png");
assert_eq!(file.data, [0x89, b'P', b'N', b'G', 0, 0xff]);
assert_eq!(calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn a_response_larger_than_the_limit_fails() {
let server = FakeServer::start(|_| async {
Response {
status: 200,
headers: vec![(
"Content-Type".to_owned(),
"application/octet-stream".to_owned(),
)],
body: vec![0; (32 << 20) + 1],
}
})
.await;
let err = service(&server)
.call_http_body(Request::new(Method::GET, "/x:download").idempotent())
.await
.unwrap_err();
assert!(matches!(err, Error::Decode(_)), "{err:?}");
assert!(err.to_string().contains("larger than the 32 MiB"), "{err}");
}
#[tokio::test]
async fn a_name_of_the_wrong_form_is_refused_before_sending() {
let err =
origincloud_core::path_value("name", "vol-1", "projects/*/zones/*/volumes/*").unwrap_err();
assert!(matches!(err, Error::InvalidRequest(_)));
assert!(
err.to_string()
.contains("name must have the form projects/*/zones/*/volumes/*"),
"{err}"
);
}
fn operation(done: bool, response: Option<Value>) -> Value {
let mut op = json!({ "name": OPERATION_NAME });
if done {
op["done"] = json!(true);
}
if let Some(response) = response {
op["response"] = response;
}
op
}
fn running(service: &Service) -> Operation<Value> {
Operation::new(
service.clone(),
serde_json::from_value(operation(false, None)).unwrap(),
"x.Volume",
)
}
#[tokio::test]
async fn wait_waits_until_the_operation_is_done() {
let waits = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&waits);
let server = FakeServer::start(move |r| {
assert_eq!(r.method, "POST");
assert_eq!(r.path, format!("/v1beta1/{OPERATION_NAME}:wait"));
assert_eq!(r.text(), r#"{"timeout":"60s"}"#);
let n = counted.fetch_add(1, Ordering::SeqCst) + 1;
async move {
if n < 3 {
Response::json(operation(false, None).to_string())
} else {
let result = json!({"@type": "type.googleapis.com/x.Volume", "name": "projects/p/zones/z/volumes/vol-1"});
Response::json(operation(true, Some(result)).to_string())
.with_header(CONSISTENCY_TOKEN_HEADER, "v1.done")
}
}
})
.await;
let client = client(&server, 5);
let mut op = running(&client.service("pvs", true));
let got = op.wait().await.unwrap();
assert_eq!(got["name"], "projects/p/zones/z/volumes/vol-1");
assert_eq!(waits.load(Ordering::SeqCst), 3);
assert!(op.done());
assert_eq!(client.consistency_token(), "v1.done");
}
#[tokio::test]
async fn a_failed_operation_returns_its_error() {
let server = FakeServer::start(|_| async {
Response::json(
json!({
"name": OPERATION_NAME,
"done": true,
"error": {
"code": 13,
"message": "the storage couldn't be created",
"details": [{"@type": "type.googleapis.com/google.rpc.ErrorInfo", "reason": "INTERNAL_SERVER_ERROR", "domain": "origincloudapis.eu", "metadata": {"zone": "z"}}],
},
})
.to_string(),
)
})
.await;
let mut op = running(&service(&server));
let err = op.wait().await.unwrap_err();
let api = err.api().expect("an API error");
assert_eq!(api.status, "INTERNAL");
assert_eq!(api.reason, "INTERNAL_SERVER_ERROR");
assert_eq!(api.metadata.get("zone").map(String::as_str), Some("z"));
assert_eq!(api.message, "the storage couldn't be created");
assert_eq!(api.http_status, None);
}
#[tokio::test]
async fn an_operation_answered_done_is_not_waited_for() {
let server =
FakeServer::start(|r| async move { panic!("unexpected call {} {}", r.method, r.path) })
.await;
let done: longrunning::Operation = serde_json::from_value(operation(
true,
Some(json!({"@type": "type.googleapis.com/google.protobuf.Empty", "value": {}})),
))
.unwrap();
let mut op: Operation<origincloud_core::protobuf::Empty> =
Operation::new(service(&server), done, "google.protobuf.Empty");
op.wait().await.unwrap();
}
#[tokio::test]
async fn the_result_of_an_operation_that_isnt_done() {
let server = FakeServer::start(|_| async { Response::json("{}") }).await;
let op = running(&service(&server));
assert!(matches!(op.result(), Err(Error::NotDone(_))));
}
#[tokio::test]
async fn a_result_of_another_type_is_refused() {
let server = FakeServer::start(|_| async { Response::json("{}") }).await;
let done = serde_json::from_value(operation(
true,
Some(json!({"@type": "type.googleapis.com/x.Snapshot", "name": "n"})),
))
.unwrap();
let op: Operation<Value> = Operation::new(service(&server), done, "x.Volume");
let err = op.result().unwrap_err();
assert!(
err.to_string().contains("is a x.Snapshot, not a x.Volume"),
"{err}"
);
}
fn done_volume() -> String {
let result = json!({"@type": "type.googleapis.com/x.Volume", "name": "projects/p/zones/z/volumes/vol-1"});
operation(true, Some(result)).to_string()
}
#[tokio::test]
async fn a_wait_call_that_times_out_counts_as_not_done() {
let waits = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&waits);
let server = FakeServer::start(move |_| {
let n = counted.fetch_add(1, Ordering::SeqCst) + 1;
async move {
if n < 3 {
tokio::time::sleep(Duration::from_secs(5)).await;
}
Response::json(done_volume())
}
})
.await;
let http = reqwest::Client::builder()
.timeout(Duration::from_millis(300))
.build()
.unwrap();
let client = Client::builder()
.endpoint(server.url())
.credentials(AccessToken::new("t"))
.http_client(http)
.max_attempts(1)
.build()
.unwrap();
let mut op = running(&client.service("pvs", true));
let got = op.wait().await.unwrap();
assert_eq!(got["name"], "projects/p/zones/z/volumes/vol-1");
assert_eq!(waits.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn the_wait_per_call_is_configurable_and_kept_below_the_timeout() {
let bodies = Arc::new(Mutex::new(Vec::<String>::new()));
let log = Arc::clone(&bodies);
let server = FakeServer::start(move |r| {
log.lock().unwrap().push(r.text());
async { Response::json(done_volume()) }
})
.await;
let cases: [(Option<u64>, Option<u64>, &str); 5] = [
(None, None, "60s"),
(Some(10_000), None, "10s"),
(None, Some(5_000), "4s"),
(Some(10_000), Some(3_000), "2s"),
(None, Some(1_500), "1s"),
];
for (wait, timeout, _) in cases {
let mut builder = Client::builder()
.endpoint(server.url())
.credentials(AccessToken::new("t"));
if let Some(ms) = wait {
builder = builder.operation_wait(Duration::from_millis(ms));
}
if let Some(ms) = timeout {
builder = builder.timeout(Duration::from_millis(ms));
}
let client = builder.build().unwrap();
running(&client.service("pvs", true)).wait().await.unwrap();
}
let want: Vec<String> = cases
.iter()
.map(|(_, _, w)| format!(r#"{{"timeout":"{w}"}}"#))
.collect();
assert_eq!(*bodies.lock().unwrap(), want);
}
#[tokio::test]
async fn an_attempt_longer_than_the_timeout_fails_and_is_retried() {
let calls = Arc::new(AtomicU32::new(0));
let counted = Arc::clone(&calls);
let server = FakeServer::start(move |_| {
let n = counted.fetch_add(1, Ordering::SeqCst) + 1;
async move {
if n == 1 {
tokio::time::sleep(Duration::from_secs(5)).await;
}
Response::json("{}")
}
})
.await;
let client = Client::builder()
.endpoint(server.url())
.credentials(AccessToken::new("t"))
.timeout(Duration::from_millis(300))
.build()
.unwrap();
get(&client.service("pvs", true), "/x").await.unwrap();
assert_eq!(calls.load(Ordering::SeqCst), 2);
}