1use crate::{
7 AccountID, Admission, ClientRequestID, Environment, Error, GenericRejection, OperationError,
8 Result, Supplied, admission::REQUESTS_PER_SECOND, models::AcceptDatetimeFormat,
9};
10use futures_util::StreamExt;
11use reqwest::{Method, StatusCode, Url};
12use serde::{Serialize, de::DeserializeOwned};
13use std::{
14 collections::HashSet,
15 fmt,
16 ops::Deref,
17 sync::{Arc, Mutex},
18 time::Duration,
19};
20
21const DEFAULT_MAX_BODY: usize = 8 * 1024 * 1024;
22const DEFAULT_COOLDOWN: Duration = Duration::from_secs(1);
23const MAX_COOLDOWN: Duration = Duration::from_mins(1);
24
25#[derive(Debug, Clone)]
27pub struct ApiResponse<T> {
28 pub body: T,
30 pub request_id: Supplied<String>,
34 pub next_page: Supplied<Url>,
40}
41
42impl<T> Deref for ApiResponse<T> {
43 type Target = T;
44 fn deref(&self) -> &Self::Target {
45 &self.body
46 }
47}
48
49#[derive(Clone)]
51pub struct Client {
52 pub(crate) inner: Arc<Inner>,
53}
54
55impl fmt::Debug for Client {
56 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
57 f.debug_struct("Client")
58 .field("environment", &self.inner.environment)
59 .finish_non_exhaustive()
60 }
61}
62
63pub(crate) struct Inner {
64 pub environment: Environment,
65 pub rest: Url,
66 pub stream: Url,
67 pub http: reqwest::Client,
68 pub stream_http: reqwest::Client,
69 pub token: String,
70 pub max_body: usize,
71 pub datetime_format: AcceptDatetimeFormat,
72 pub(crate) admission: Admission,
73 fenced: Mutex<HashSet<String>>,
74 in_flight: Mutex<HashSet<String>>,
75}
76
77#[derive(Clone, Copy)]
78pub(crate) enum MutationAdmission<'a> {
79 Ordinary(&'a AccountID),
80 Exit(&'a AccountID),
81}
82
83impl<'a> MutationAdmission<'a> {
84 fn account(self) -> &'a AccountID {
85 match self {
86 Self::Ordinary(account) | Self::Exit(account) => account,
87 }
88 }
89}
90
91pub struct ClientBuilder {
93 environment: Environment,
94 token: String,
95 rest_override: Option<Url>,
96 stream_override: Option<Url>,
97 max_body: usize,
98 timeout: Duration,
99 exit_reserve: Option<u32>,
100 admission: Option<Admission>,
101 datetime_format: AcceptDatetimeFormat,
102}
103
104impl fmt::Debug for ClientBuilder {
105 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
106 f.debug_struct("ClientBuilder")
107 .field("environment", &self.environment)
108 .field("max_body", &self.max_body)
109 .finish_non_exhaustive()
110 }
111}
112
113impl Client {
114 pub fn set_exit_reserve(&self, requests: u32) -> Result<()> {
126 self.inner.admission.set_exit_reserve(requests)
127 }
128
129 pub fn builder(environment: Environment, bearer_token: impl Into<String>) -> ClientBuilder {
131 ClientBuilder {
132 environment,
133 token: bearer_token.into(),
134 rest_override: None,
135 stream_override: None,
136 max_body: DEFAULT_MAX_BODY,
137 timeout: Duration::from_secs(30),
138 exit_reserve: None,
139 admission: None,
140 datetime_format: AcceptDatetimeFormat::Rfc3339,
141 }
142 }
143
144 pub fn acknowledge_reconciliation(&self, account: &AccountID) -> Result<()> {
153 let mut fenced = self
154 .inner
155 .fenced
156 .lock()
157 .map_err(|_| Error::InvalidInput("reconciliation lock poisoned".into()))?;
158 fenced.remove(account.as_str());
159 Ok(())
160 }
161
162 #[allow(clippy::too_many_lines)]
164 pub(crate) async fn execute<
165 T: DeserializeOwned,
166 R: DeserializeOwned,
167 Q: Serialize,
168 B: Serialize,
169 >(
170 &self,
171 method: Method,
172 path: &str,
173 query: Option<&Q>,
174 body: Option<&B>,
175 mutation: Option<MutationAdmission<'_>>,
176 client_request_id: Option<&ClientRequestID>,
177 ) -> std::result::Result<ApiResponse<T>, OperationError<R>> {
178 let mutation_account = mutation.map(MutationAdmission::account);
179 let exit = matches!(mutation, Some(MutationAdmission::Exit(_)));
180 let query_value = query
181 .map(serde_json::to_value)
182 .transpose()
183 .map_err(|e| Error::InvalidInput(e.to_string()))?;
184 let body_value = body
185 .map(serde_json::to_value)
186 .transpose()
187 .map_err(|e| Error::InvalidInput(e.to_string()))?;
188 crate::validation::validate(path, query_value.as_ref(), body_value.as_ref())?;
189 self.admit(mutation_account.is_some(), exit).await?;
190 let url = Self::build_url(&self.inner.rest, path, query)?;
191 let mut request = self
192 .inner
193 .http
194 .request(method, url)
195 .bearer_auth(&self.inner.token)
196 .header(
197 "Accept-Datetime-Format",
198 self.inner.datetime_format.as_str(),
199 );
200 if let Some(id) = client_request_id {
201 request = request.header("ClientRequestID", id.as_str());
202 }
203 if let Some(body) = body {
204 request = request.json(body);
205 }
206 let mut guard = mutation_account
209 .map(|account| MutationGuard::new(self.inner.clone(), account))
210 .transpose()?;
211 let response = request.send().await.map_err(|e| match mutation_account {
212 Some(account) => Error::AmbiguousMutation {
213 account: account.to_string(),
214 },
215 None => Error::Transport(e.to_string()),
216 })?;
217 let status = response.status();
218 let request_id = supplied_header(response.headers(), "RequestID");
219 let next_page = if status.is_success() {
220 parse_next_page(&self.inner.rest, response.headers()).map_err(|error| {
221 match mutation_account {
222 Some(account) => Error::AmbiguousMutation {
223 account: account.to_string(),
224 },
225 None => error,
226 }
227 })?
228 } else {
229 Supplied::Absent
230 };
231 if status == StatusCode::TOO_MANY_REQUESTS {
232 self.install_cooldown(response.headers().get(reqwest::header::RETRY_AFTER));
233 if let Some(ref mut guard) = guard {
234 guard.disarm();
235 }
236 return Err(Error::ProviderRateLimited.into());
237 }
238 let bytes = read_bounded(response, self.inner.max_body)
239 .await
240 .map_err(|e| match mutation_account {
241 Some(account) => Error::AmbiguousMutation {
242 account: account.to_string(),
243 },
244 None => e,
245 })?;
246 if status.is_success() {
247 let value = serde_json::from_slice(&bytes).map_err(|e| match mutation_account {
248 Some(account) => Error::AmbiguousMutation {
249 account: account.to_string(),
250 },
251 None => Error::Decode(e.to_string()),
252 })?;
253 if let Some(ref mut guard) = guard {
254 guard.disarm();
255 }
256 return Ok(ApiResponse {
257 body: value,
258 request_id,
259 next_page,
260 });
261 }
262 if mutation_account.is_some()
263 && (status.is_server_error()
264 || status == StatusCode::REQUEST_TIMEOUT
265 || !matches!(status.as_u16(), 400 | 401 | 403 | 404 | 405))
266 {
267 return Err(Error::AmbiguousMutation {
268 account: mutation_account.map_or(String::new(), ToString::to_string),
269 }
270 .into());
271 }
272 let body: Supplied<R> = if bytes.iter().all(u8::is_ascii_whitespace) {
273 Supplied::Absent
274 } else {
275 match serde_json::from_slice::<R>(&bytes) {
276 Ok(value) => Supplied::Decoded(value),
277 Err(_) => Supplied::Undecoded,
278 }
279 };
280 let reason = ProviderError::read(&bytes);
286 if let Some(ref mut guard) = guard {
287 guard.disarm();
288 }
289 Err(OperationError::Rejected {
290 status: status.as_u16(),
291 code: reason.error_code,
292 message: reason.error_message,
293 body,
294 })
295 }
296
297 pub async fn fetch_page<T: DeserializeOwned>(
305 &self,
306 next_page: &Url,
307 ) -> std::result::Result<ApiResponse<T>, OperationError<GenericRejection>> {
308 if next_page.scheme() != self.inner.rest.scheme()
309 || next_page.host_str() != self.inner.rest.host_str()
310 || next_page.port_or_known_default() != self.inner.rest.port_or_known_default()
311 || !next_page.path().starts_with("/v3/")
312 || next_page.fragment().is_some()
313 || next_page.username() != ""
314 || next_page.password().is_some()
315 {
316 return Err(Error::InvalidInput(
317 "pagination URL is outside the OANDA REST origin".into(),
318 )
319 .into());
320 }
321 let path = if let Some(query) = next_page.query() {
322 format!("{}?{query}", next_page.path())
323 } else {
324 next_page.path().to_owned()
325 };
326 self.execute(Method::GET, &path, None::<&()>, None::<&()>, None, None)
327 .await
328 }
329
330 pub(crate) async fn admit(&self, mutation: bool, exit: bool) -> Result<()> {
331 self.inner.admission.admit(mutation, exit).await
332 }
333
334 pub(crate) fn build_url<Q: Serialize>(
335 base: &Url,
336 path: &str,
337 query: Option<&Q>,
338 ) -> Result<Url> {
339 if !path.starts_with("/v3/") {
340 return Err(Error::InvalidInput("path must start with /v3/".into()));
341 }
342 let mut url = base
343 .join(path)
344 .map_err(|e| Error::InvalidInput(e.to_string()))?;
345 if let Some(query) = query {
346 let value =
347 serde_json::to_value(query).map_err(|e| Error::InvalidInput(e.to_string()))?;
348 let object = value
349 .as_object()
350 .ok_or_else(|| Error::InvalidInput("query must be an object".into()))?;
351 let mut pairs = url.query_pairs_mut();
352 for (key, value) in object {
353 if value.is_null() {
354 continue;
355 }
356 if let Some(items) = value.as_array() {
357 let csv = items
358 .iter()
359 .map(query_atom)
360 .collect::<Result<Vec<_>>>()?
361 .join(",");
362 pairs.append_pair(key, &csv);
363 } else {
364 pairs.append_pair(key, &query_atom(value)?);
365 }
366 }
367 }
368 Ok(url)
369 }
370
371 fn install_cooldown(&self, retry_after: Option<&reqwest::header::HeaderValue>) {
378 let cooldown = match retry_after {
379 None => DEFAULT_COOLDOWN,
380 Some(value) => parse_retry_after(value).unwrap_or(MAX_COOLDOWN),
381 };
382 self.inner.admission.install_cooldown(cooldown);
383 }
384}
385
386fn parse_retry_after(value: &reqwest::header::HeaderValue) -> Option<Duration> {
391 let text = value.to_str().ok()?.trim();
392 if let Ok(seconds) = text.parse::<u64>() {
393 return Some(Duration::from_secs(seconds).min(MAX_COOLDOWN));
394 }
395 let when = chrono::DateTime::parse_from_rfc2822(text).ok()?;
396 let seconds = u64::try_from(
397 (when.with_timezone(&chrono::Utc) - chrono::Utc::now())
398 .num_seconds()
399 .max(0),
400 )
401 .ok()?;
402 Some(Duration::from_secs(seconds).min(MAX_COOLDOWN))
403}
404
405impl ClientBuilder {
406 #[must_use]
408 pub fn max_response_bytes(mut self, value: usize) -> Self {
409 self.max_body = value;
410 self
411 }
412
413 #[must_use]
420 pub fn exit_reserve(mut self, requests: u32) -> Self {
421 self.exit_reserve = Some(requests);
422 self
423 }
424
425 #[must_use]
430 pub fn admission(mut self, owner: Admission) -> Self {
431 self.admission = Some(owner);
432 self
433 }
434
435 #[must_use]
437 pub fn timeout(mut self, value: Duration) -> Self {
438 self.timeout = value;
439 self
440 }
441 #[must_use]
443 pub fn datetime_format(mut self, value: AcceptDatetimeFormat) -> Self {
444 self.datetime_format = value;
445 self
446 }
447 #[must_use]
450 pub fn endpoints(mut self, rest: Url, stream: Url) -> Self {
451 self.rest_override = Some(rest);
452 self.stream_override = Some(stream);
453 self
454 }
455
456 pub fn build(self) -> Result<Client> {
463 if self.token.trim().is_empty() || self.token.chars().any(char::is_control) {
464 return Err(Error::InvalidInput("invalid bearer token".into()));
465 }
466 if self.max_body == 0 || self.timeout.is_zero() {
467 return Err(Error::InvalidInput("limits must be positive".into()));
468 }
469 if self
470 .exit_reserve
471 .is_some_and(|reserve| reserve >= REQUESTS_PER_SECOND)
472 {
473 return Err(Error::InvalidInput(
474 "exit reserve must be less than 120 requests".into(),
475 ));
476 }
477 let admission = match self.admission {
478 Some(owner) => {
479 if let Some(requested) = self.exit_reserve
480 && requested != owner.exit_reserve()?
481 {
482 return Err(Error::InvalidInput(
483 "exit reserve conflicts with shared admission policy".into(),
484 ));
485 }
486 owner
487 }
488 None => Admission::with_exit_reserve(self.exit_reserve.unwrap_or(0))?,
489 };
490 let (default_rest, default_stream) = self.environment.urls()?;
491 let rest = self.rest_override.unwrap_or_else(|| default_rest.clone());
492 let stream = self
493 .stream_override
494 .unwrap_or_else(|| default_stream.clone());
495 validate_url(&rest)?;
496 validate_url(&stream)?;
497 let paired_loopback = is_loopback(&rest) && is_loopback(&stream);
498 if !paired_loopback && (rest != default_rest || stream != default_stream) {
499 return Err(Error::InvalidInput(
500 "endpoints must use the selected OANDA environment pair or loopback fixtures"
501 .into(),
502 ));
503 }
504 if matches!(self.datetime_format, AcceptDatetimeFormat::Unknown(_)) {
505 return Err(Error::InvalidInput("unsupported datetime format".into()));
506 }
507 let http = reqwest::Client::builder()
508 .connector_layer(admission.connection_layer())
509 .timeout(self.timeout)
510 .redirect(reqwest::redirect::Policy::none())
511 .retry(reqwest::retry::never())
512 .no_proxy()
513 .build()
514 .map_err(|e| Error::InvalidInput(e.to_string()))?;
515 let stream_http = stream_transport(reqwest::Client::builder(), self.timeout)
516 .connector_layer(admission.connection_layer())
517 .build()
518 .map_err(|e| Error::InvalidInput(e.to_string()))?;
519 Ok(Client {
520 inner: Arc::new(Inner {
521 environment: self.environment,
522 rest,
523 stream,
524 http,
525 stream_http,
526 token: self.token,
527 max_body: self.max_body,
528 datetime_format: self.datetime_format,
529 admission,
530 fenced: Mutex::new(HashSet::new()),
531 in_flight: Mutex::new(HashSet::new()),
532 }),
533 })
534 }
535}
536
537fn validate_url(url: &Url) -> Result<()> {
538 let host = url
539 .host_str()
540 .ok_or_else(|| Error::InvalidInput("endpoint must have a host".into()))?;
541 let loopback = host == "localhost" || host == "127.0.0.1" || host == "::1";
542 if url.scheme() != "https" && !(loopback && url.scheme() == "http") {
543 return Err(Error::InvalidInput("remote endpoints require HTTPS".into()));
544 }
545 if url.path() != "/"
546 || url.username() != ""
547 || url.password().is_some()
548 || url.query().is_some()
549 || url.fragment().is_some()
550 {
551 return Err(Error::InvalidInput(
552 "endpoint URL must not contain credentials, query, or fragment".into(),
553 ));
554 }
555 Ok(())
556}
557
558fn is_loopback(url: &Url) -> bool {
559 matches!(url.host_str(), Some("localhost" | "127.0.0.1" | "::1"))
560}
561
562fn query_atom(value: &serde_json::Value) -> Result<String> {
563 match value {
564 serde_json::Value::String(s) => Ok(s.clone()),
565 serde_json::Value::Number(n) => Ok(n.to_string()),
566 serde_json::Value::Bool(b) => Ok(b.to_string()),
567 _ => Err(Error::InvalidInput("unsupported query value".into())),
568 }
569}
570
571fn supplied_header(headers: &reqwest::header::HeaderMap, name: &str) -> Supplied<String> {
577 match headers.get(name) {
578 None => Supplied::Absent,
579 Some(value) => match value.to_str() {
580 Ok(text) => Supplied::Decoded(text.to_owned()),
581 Err(_) => Supplied::Undecoded,
582 },
583 }
584}
585
586fn parse_next_page(base: &Url, headers: &reqwest::header::HeaderMap) -> Result<Supplied<Url>> {
595 let Some(link) = headers.get(reqwest::header::LINK) else {
596 return Ok(Supplied::Absent);
597 };
598 let Ok(value) = link.to_str() else {
599 return Ok(Supplied::Undecoded);
600 };
601 let Some(item) = value
602 .split(',')
603 .find(|item| item.contains("rel=\"next\"") || item.contains("rel=next"))
604 else {
605 return Ok(Supplied::Absent);
606 };
607 let Some(start) = item.find('<') else {
613 return Err(Error::Decode("invalid pagination link".into()));
614 };
615 let Some(end) = item[start + 1..].find('>') else {
616 return Err(Error::Decode("invalid pagination link".into()));
617 };
618 let url = base
619 .join(&item[start + 1..start + 1 + end])
620 .map_err(|e| Error::Decode(e.to_string()))?;
621 if url.scheme() != base.scheme()
622 || url.host_str() != base.host_str()
623 || url.port_or_known_default() != base.port_or_known_default()
624 || !url.path().starts_with("/v3/")
625 {
626 return Err(Error::Decode(
627 "pagination link changed origin or API version".into(),
628 ));
629 }
630 Ok(Supplied::Decoded(url))
631}
632
633pub(crate) fn segment(value: &str) -> String {
634 const PATH_SEGMENT: &percent_encoding::AsciiSet = &percent_encoding::NON_ALPHANUMERIC
635 .remove(b'-')
636 .remove(b'_')
637 .remove(b'.')
638 .remove(b'~');
639 percent_encoding::utf8_percent_encode(value, PATH_SEGMENT).to_string()
640}
641
642pub(crate) async fn read_bounded(response: reqwest::Response, max: usize) -> Result<Vec<u8>> {
643 let mut stream = response.bytes_stream();
644 let mut bytes = Vec::new();
645 while let Some(next) = stream.next().await {
646 let chunk = next.map_err(|e| Error::Transport(e.to_string()))?;
647 if bytes.len().saturating_add(chunk.len()) > max {
648 return Err(Error::ResponseTooLarge);
649 }
650 bytes.extend_from_slice(&chunk);
651 }
652 Ok(bytes)
653}
654
655#[derive(serde::Deserialize)]
656#[serde(rename_all = "camelCase")]
657struct ProviderError {
658 error_code: Option<String>,
659 error_message: Option<String>,
660}
661
662impl ProviderError {
663 fn read(bytes: &[u8]) -> Self {
667 let value: serde_json::Value = match serde_json::from_slice(bytes) {
668 Ok(value) => value,
669 Err(_) => return Self::UNREAD,
670 };
671 Self {
672 error_code: value
673 .get("errorCode")
674 .and_then(serde_json::Value::as_str)
675 .map(str::to_owned),
676 error_message: value
677 .get("errorMessage")
678 .and_then(serde_json::Value::as_str)
679 .map(str::to_owned),
680 }
681 }
682
683 const UNREAD: Self = Self {
687 error_code: None,
688 error_message: None,
689 };
690}
691
692struct MutationGuard {
693 inner: Arc<Inner>,
694 account: String,
695 armed: bool,
696}
697impl MutationGuard {
698 fn new(inner: Arc<Inner>, account: &AccountID) -> Result<Self> {
699 if inner
700 .fenced
701 .lock()
702 .map_err(|_| Error::InvalidInput("reconciliation lock poisoned".into()))?
703 .contains(account.as_str())
704 {
705 return Err(Error::ReconciliationRequired {
706 account: account.to_string(),
707 });
708 }
709 let mut in_flight = inner
710 .in_flight
711 .lock()
712 .map_err(|_| Error::InvalidInput("mutation lock poisoned".into()))?;
713 if !in_flight.insert(account.to_string()) {
714 return Err(Error::MutationInFlight {
715 account: account.to_string(),
716 });
717 }
718 if inner
721 .fenced
722 .lock()
723 .map_err(|_| Error::InvalidInput("reconciliation lock poisoned".into()))?
724 .contains(account.as_str())
725 {
726 in_flight.remove(account.as_str());
727 return Err(Error::ReconciliationRequired {
728 account: account.to_string(),
729 });
730 }
731 drop(in_flight);
732 Ok(Self {
733 inner,
734 account: account.to_string(),
735 armed: true,
736 })
737 }
738 fn disarm(&mut self) {
739 self.armed = false;
740 }
741}
742impl Drop for MutationGuard {
743 fn drop(&mut self) {
744 if self.armed {
749 let mut fenced = self
750 .inner
751 .fenced
752 .lock()
753 .unwrap_or_else(std::sync::PoisonError::into_inner);
754 fenced.insert(self.account.clone());
755 }
756 let mut in_flight = self
757 .inner
758 .in_flight
759 .lock()
760 .unwrap_or_else(std::sync::PoisonError::into_inner);
761 in_flight.remove(&self.account);
762 }
763}
764
765fn stream_transport(builder: reqwest::ClientBuilder, timeout: Duration) -> reqwest::ClientBuilder {
766 builder
767 .http1_only()
768 .connect_timeout(timeout)
769 .redirect(reqwest::redirect::Policy::none())
770 .retry(reqwest::retry::never())
771 .no_proxy()
772}
773
774#[cfg(test)]
775#[allow(clippy::expect_used, clippy::panic)]
778mod tests {
779 use super::*;
780 use tokio::time::Instant as RateInstant;
781
782 #[tokio::test]
783 async fn streaming_uses_http11_when_the_caller_enables_http2() {
784 use tokio::io::{AsyncReadExt, AsyncWriteExt};
785 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
786 .await
787 .expect("bind");
788 let address = listener.local_addr().expect("address");
789 let server = tokio::spawn(async move {
790 let (mut socket, _) = listener.accept().await.expect("accept");
791 let mut bytes = Vec::new();
792 let mut buffer = [0; 1024];
793 loop {
794 let count = socket.read(&mut buffer).await.expect("read request");
795 if count == 0 {
796 break;
797 }
798 bytes.extend_from_slice(&buffer[..count]);
799 if bytes.windows(4).any(|window| window == b"\r\n\r\n") {
800 break;
801 }
802 }
803 socket
804 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
805 .await
806 .expect("response");
807 bytes
808 });
809 let client = stream_transport(
810 reqwest::Client::builder().http2_prior_knowledge(),
811 Duration::from_secs(5),
812 )
813 .build()
814 .expect("client");
815 let response = client
816 .get(format!(
817 "http://{address}/v3/accounts/fixture/transactions/stream"
818 ))
819 .send()
820 .await;
821 let request = server.await.expect("server");
822 assert!(
823 request.starts_with(b"GET /v3/accounts/fixture/transactions/stream HTTP/1.1\r\n"),
824 "request protocol: {}",
825 String::from_utf8_lossy(&request)
826 );
827 assert_eq!(
828 response.expect("stream response").version(),
829 reqwest::Version::HTTP_11
830 );
831 }
832
833 #[tokio::test]
834 async fn mutation_rate_admission_is_shared_by_clones() {
835 let client = Client::builder(Environment::Practice, "fixture").build();
836 assert!(client.is_ok());
837 let Ok(client) = client else { return };
838 if let Ok(mut rate) = client.inner.admission.inner.rate.lock() {
839 for _ in 0..REQUESTS_PER_SECOND {
840 rate.admitted.push_back(RateInstant::now());
841 }
842 }
843 assert!(matches!(
844 client.clone().admit(true, false).await,
845 Err(Error::RateLimited)
846 ));
847 }
848
849 #[test]
850 fn account_reservation_and_reconciliation_are_shared() {
851 let client = Client::builder(Environment::Practice, "fixture").build();
852 assert!(client.is_ok());
853 let Ok(client) = client else { return };
854 let account = AccountID::new("101-001-1-001");
855 assert!(account.is_ok());
856 let Ok(account) = account else { return };
857 let first = MutationGuard::new(client.inner.clone(), &account);
858 assert!(first.is_ok());
859 let Ok(first) = first else { return };
860 assert!(matches!(
861 MutationGuard::new(client.clone().inner, &account),
862 Err(Error::MutationInFlight { .. })
863 ));
864 drop(first);
865 assert!(matches!(
866 MutationGuard::new(client.inner.clone(), &account),
867 Err(Error::ReconciliationRequired { .. })
868 ));
869 let other = AccountID::new("101-001-1-002");
870 assert!(other.is_ok());
871 if let Ok(other) = other {
872 let mut independent = MutationGuard::new(client.inner.clone(), &other);
873 assert!(independent.is_ok());
874 if let Ok(ref mut guard) = independent {
875 guard.disarm();
876 }
877 }
878 client
879 .acknowledge_reconciliation(&account)
880 .expect("fence released");
881 let mut next = MutationGuard::new(client.inner.clone(), &account);
882 assert!(next.is_ok());
883 if let Ok(ref mut guard) = next {
884 guard.disarm();
885 }
886 }
887
888 fn poison<T>(lock: &Mutex<T>) {
889 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
890 let Ok(_held) = lock.lock() else {
891 return;
892 };
893 panic!("poison the lock");
894 }));
895 assert!(result.is_err());
896 assert!(lock.is_poisoned());
897 }
898
899 #[test]
900 fn an_ambiguous_mutation_is_fenced_even_after_a_poisoned_lock() {
901 let client = Client::builder(Environment::Practice, "fixture").build();
902 assert!(client.is_ok());
903 let Ok(client) = client else { return };
904 let account = AccountID::new("101-001-1-001");
905 assert!(account.is_ok());
906 let Ok(account) = account else { return };
907 let guard = MutationGuard::new(client.inner.clone(), &account);
908 assert!(guard.is_ok());
909 let Ok(guard) = guard else { return };
910 poison(&client.inner.fenced);
913 poison(&client.inner.in_flight);
914 drop(guard);
915 let fenced = client
916 .inner
917 .fenced
918 .lock()
919 .unwrap_or_else(std::sync::PoisonError::into_inner);
920 assert!(
921 fenced.contains(account.as_str()),
922 "the ambiguity must be fenced, not dropped"
923 );
924 let in_flight = client
925 .inner
926 .in_flight
927 .lock()
928 .unwrap_or_else(std::sync::PoisonError::into_inner);
929 assert!(!in_flight.contains(account.as_str()));
930 }
931
932 #[test]
933 fn a_failed_acknowledgement_is_reported_and_leaves_the_account_fenced() {
934 let client = Client::builder(Environment::Practice, "fixture").build();
935 assert!(client.is_ok());
936 let Ok(client) = client else { return };
937 let account = AccountID::new("101-001-1-001");
938 assert!(account.is_ok());
939 let Ok(account) = account else { return };
940 {
941 let mut fenced = client
942 .inner
943 .fenced
944 .lock()
945 .unwrap_or_else(std::sync::PoisonError::into_inner);
946 fenced.insert(account.to_string());
947 }
948 poison(&client.inner.fenced);
949 assert!(matches!(
950 client.acknowledge_reconciliation(&account),
951 Err(Error::InvalidInput(_))
952 ));
953 let fenced = client
954 .inner
955 .fenced
956 .lock()
957 .unwrap_or_else(std::sync::PoisonError::into_inner);
958 assert!(fenced.contains(account.as_str()));
959 }
960
961 fn assert_cooldown(header: Option<&str>, expected: Duration) {
962 let client = Client::builder(Environment::Practice, "fixture").build();
963 assert!(client.is_ok());
964 let Ok(client) = client else { return };
965 poison(&client.inner.admission.inner.rate);
966 let header =
967 header.map(|text| reqwest::header::HeaderValue::from_str(text).expect("header"));
968 let start = RateInstant::now();
969 client.install_cooldown(header.as_ref());
970 let rate = client
971 .inner
972 .admission
973 .inner
974 .rate
975 .lock()
976 .unwrap_or_else(std::sync::PoisonError::into_inner);
977 let until = rate.cooldown_until.expect("cooldown installed");
978 let tolerance = Duration::from_secs(1);
983 assert!(
984 until + tolerance >= start + expected,
985 "cooldown shorter than {expected:?}"
986 );
987 assert!(
988 until <= RateInstant::now() + expected + tolerance,
989 "cooldown longer than {expected:?}"
990 );
991 }
992
993 #[test]
994 fn retry_after_reads_both_rfc_9110_forms() {
995 assert_cooldown(Some("7"), Duration::from_secs(7));
996 let later = chrono::Utc::now() + chrono::Duration::seconds(30);
998 assert_cooldown(
999 Some(&later.format("%a, %d %b %Y %H:%M:%S GMT").to_string()),
1000 Duration::from_secs(29),
1001 );
1002 let earlier = chrono::Utc::now() - chrono::Duration::seconds(30);
1004 assert_cooldown(
1005 Some(&earlier.format("%a, %d %b %Y %H:%M:%S GMT").to_string()),
1006 Duration::ZERO,
1007 );
1008 assert_cooldown(Some("9999"), MAX_COOLDOWN);
1009 }
1010
1011 #[test]
1012 fn an_unreadable_or_absent_retry_after_is_not_read_as_a_provider_instruction() {
1013 assert_cooldown(None, DEFAULT_COOLDOWN);
1015 assert_cooldown(Some("soon"), MAX_COOLDOWN);
1018 }
1019
1020 fn reserved_client() -> Client {
1021 Client::builder(Environment::Practice, "fixture")
1022 .exit_reserve(2)
1023 .build()
1024 .expect("valid reserve")
1025 }
1026
1027 #[test]
1028 fn exit_reserve_rejects_the_full_window_and_larger_values() {
1029 for reserve in [120, 121, u32::MAX] {
1030 assert!(matches!(
1031 Client::builder(Environment::Practice, "fixture")
1032 .exit_reserve(reserve)
1033 .build(),
1034 Err(Error::InvalidInput(_))
1035 ));
1036 }
1037 }
1038
1039 #[tokio::test(start_paused = true)]
1040 async fn ordinary_admission_leaves_two_exits_and_preserves_the_aggregate_limit() {
1041 let client = reserved_client();
1042 for _ in 0..118 {
1043 client.admit(false, false).await.expect("ordinary slot");
1044 }
1045 assert!(matches!(
1046 client.admit(true, false).await,
1047 Err(Error::RateLimited)
1048 ));
1049 let read = client.admit(false, false);
1050 tokio::pin!(read);
1051 assert!(futures_util::poll!(read.as_mut()).is_pending());
1052 client.clone().admit(true, true).await.expect("first exit");
1053 client.clone().admit(true, true).await.expect("second exit");
1054 assert!(matches!(
1055 client.admit(true, true).await,
1056 Err(Error::RateLimited)
1057 ));
1058 assert!(futures_util::poll!(read.as_mut()).is_pending());
1059 tokio::time::advance(Duration::from_millis(999)).await;
1060 assert!(futures_util::poll!(read.as_mut()).is_pending());
1061 tokio::time::advance(Duration::from_millis(1)).await;
1062 read.await.expect("ordinary slot after exact expiry");
1063 }
1064
1065 #[tokio::test(start_paused = true)]
1066 async fn concurrent_clones_cannot_exceed_the_two_remaining_exit_slots() {
1067 let client = reserved_client();
1068 for _ in 0..118 {
1069 client.admit(false, false).await.expect("ordinary slot");
1070 }
1071 let mut tasks = Vec::new();
1072 for _ in 0..20 {
1073 let clone = client.clone();
1074 tasks.push(tokio::spawn(async move { clone.admit(true, true).await }));
1075 }
1076 let mut admitted = 0;
1077 let mut refused = 0;
1078 for task in tasks {
1079 match task.await.expect("join") {
1080 Ok(()) => admitted += 1,
1081 Err(Error::RateLimited) => refused += 1,
1082 result => panic!("unexpected admission: {result:?}"),
1083 }
1084 }
1085 assert_eq!(admitted, 2);
1086 assert_eq!(refused, 18);
1087 }
1088
1089 #[tokio::test(start_paused = true)]
1090 async fn cancelled_ordinary_waiters_do_not_charge_the_next_window() {
1091 let client = reserved_client();
1092 for _ in 0..118 {
1093 client.admit(false, false).await.expect("ordinary slot");
1094 }
1095 {
1096 let read = client.admit(false, false);
1097 tokio::pin!(read);
1098 assert!(futures_util::poll!(read.as_mut()).is_pending());
1099 }
1100 tokio::time::advance(Duration::from_secs(1)).await;
1101 for _ in 0..118 {
1102 client.admit(true, false).await.expect("new ordinary slot");
1103 }
1104 assert!(matches!(
1105 client.admit(true, false).await,
1106 Err(Error::RateLimited)
1107 ));
1108 client.admit(true, true).await.expect("first exit");
1109 client.admit(true, true).await.expect("second exit");
1110 }
1111
1112 #[tokio::test(start_paused = true)]
1113 async fn provider_cooldown_blocks_both_classes_until_exact_expiry() {
1114 let client = reserved_client();
1115 client.install_cooldown(Some(&reqwest::header::HeaderValue::from_static("2")));
1116 assert!(matches!(
1117 client.clone().admit(true, false).await,
1118 Err(Error::RateLimited)
1119 ));
1120 assert!(matches!(
1121 client.clone().admit(true, true).await,
1122 Err(Error::RateLimited)
1123 ));
1124 let read = client.admit(false, false);
1125 tokio::pin!(read);
1126 assert!(futures_util::poll!(read.as_mut()).is_pending());
1127 tokio::time::advance(Duration::from_millis(1999)).await;
1128 assert!(futures_util::poll!(read.as_mut()).is_pending());
1129 assert!(matches!(
1130 client.admit(true, true).await,
1131 Err(Error::RateLimited)
1132 ));
1133 tokio::time::advance(Duration::from_millis(1)).await;
1134 read.await.expect("read after cooldown");
1135 client.admit(true, true).await.expect("exit after cooldown");
1136 }
1137
1138 #[tokio::test(start_paused = true)]
1139 async fn a_shorter_provider_cooldown_cannot_release_an_existing_deadline_early() {
1140 let client = reserved_client();
1141 client.install_cooldown(Some(&reqwest::header::HeaderValue::from_static("5")));
1142 tokio::time::advance(Duration::from_secs(1)).await;
1143 client.install_cooldown(Some(&reqwest::header::HeaderValue::from_static("1")));
1144 tokio::time::advance(Duration::from_secs(1)).await;
1145 assert!(matches!(
1146 client.admit(true, true).await,
1147 Err(Error::RateLimited)
1148 ));
1149 assert!(matches!(
1150 client.admit(true, false).await,
1151 Err(Error::RateLimited)
1152 ));
1153 tokio::time::advance(Duration::from_secs(3)).await;
1154 client
1155 .admit(true, true)
1156 .await
1157 .expect("original cooldown expired");
1158 }
1159
1160 #[tokio::test(start_paused = true)]
1161 async fn runtime_release_wakes_all_registered_waiters_without_advancing_time() {
1162 let client = reserved_client();
1163 for _ in 0..118 {
1164 client.admit(false, false).await.expect("ordinary slot");
1165 }
1166 let first = client.admit(false, false);
1167 let second = client.admit(false, false);
1168 tokio::pin!(first, second);
1169 assert!(futures_util::poll!(first.as_mut()).is_pending());
1170 assert!(futures_util::poll!(second.as_mut()).is_pending());
1171 let started = RateInstant::now();
1172 client.clone().set_exit_reserve(0).expect("release reserve");
1173 assert!(matches!(
1174 futures_util::poll!(first.as_mut()),
1175 std::task::Poll::Ready(Ok(()))
1176 ));
1177 assert!(matches!(
1178 futures_util::poll!(second.as_mut()),
1179 std::task::Poll::Ready(Ok(()))
1180 ));
1181 assert_eq!(RateInstant::now(), started);
1182 assert!(matches!(
1183 client.admit(true, true).await,
1184 Err(Error::RateLimited)
1185 ));
1186 }
1187
1188 #[tokio::test(start_paused = true)]
1189 async fn runtime_growth_rechecks_waiters_and_preserves_spent_capacity() {
1190 let client = reserved_client();
1191 for _ in 0..118 {
1192 client.admit(false, false).await.expect("ordinary slot");
1193 }
1194 let read = client.admit(false, false);
1195 tokio::pin!(read);
1196 assert!(futures_util::poll!(read.as_mut()).is_pending());
1197 client.set_exit_reserve(0).expect("release reserve");
1198 client
1199 .clone()
1200 .set_exit_reserve(119)
1201 .expect("replace reserve");
1202 assert!(futures_util::poll!(read.as_mut()).is_pending());
1203 client.admit(true, true).await.expect("first exit");
1204 client.admit(true, true).await.expect("second exit");
1205 assert!(matches!(
1206 client.admit(true, true).await,
1207 Err(Error::RateLimited)
1208 ));
1209 client
1210 .set_exit_reserve(0)
1211 .expect("release exhausted reserve");
1212 client
1213 .set_exit_reserve(119)
1214 .expect("raise exhausted reserve");
1215 assert!(matches!(
1216 client.admit(true, true).await,
1217 Err(Error::RateLimited)
1218 ));
1219 client.set_exit_reserve(0).expect("release reserve");
1220 assert!(futures_util::poll!(read.as_mut()).is_pending());
1221 tokio::time::advance(Duration::from_secs(1)).await;
1222 assert!(matches!(
1223 futures_util::poll!(read.as_mut()),
1224 std::task::Poll::Ready(Ok(()))
1225 ));
1226 }
1227
1228 #[tokio::test(start_paused = true)]
1229 async fn runtime_changes_preserve_cooldown_window_and_invalid_policy_state() {
1230 let client = reserved_client();
1231 for _ in 0..118 {
1232 client.admit(false, false).await.expect("ordinary slot");
1233 }
1234 client.install_cooldown(Some(&reqwest::header::HeaderValue::from_static("2")));
1235 for reserve in [119, 0, 2] {
1236 client.set_exit_reserve(reserve).expect("replace reserve");
1237 assert!(matches!(
1238 client.admit(true, true).await,
1239 Err(Error::RateLimited)
1240 ));
1241 }
1242 for reserve in [120, u32::MAX] {
1243 assert!(matches!(
1244 client.set_exit_reserve(reserve),
1245 Err(Error::InvalidInput(_))
1246 ));
1247 let rate = client.inner.admission.inner.rate.lock().expect("rate lock");
1248 assert_eq!(rate.exit_reserve, 2);
1249 assert_eq!(rate.admitted.len(), 118);
1250 }
1251 client.set_exit_reserve(0).expect("release reserve");
1252 let read = client.admit(false, false);
1253 tokio::pin!(read);
1254 assert!(futures_util::poll!(read.as_mut()).is_pending());
1255 tokio::time::advance(Duration::from_secs(1)).await;
1256 assert!(futures_util::poll!(read.as_mut()).is_pending());
1257 tokio::time::advance(Duration::from_secs(1)).await;
1258 assert!(matches!(
1259 futures_util::poll!(read.as_mut()),
1260 std::task::Poll::Ready(Ok(()))
1261 ));
1262 for _ in 0..119 {
1263 client
1264 .admit(true, true)
1265 .await
1266 .expect("remaining aggregate capacity");
1267 }
1268 assert!(matches!(
1269 client.admit(true, true).await,
1270 Err(Error::RateLimited)
1271 ));
1272 }
1273
1274 #[tokio::test(start_paused = true)]
1275 async fn runtime_replacement_keeps_each_admission_deadline_and_independent_client() {
1276 let client = reserved_client();
1277 for _ in 0..118 {
1278 client.admit(false, false).await.expect("ordinary slot");
1279 }
1280 tokio::time::advance(Duration::from_millis(900)).await;
1281 client.set_exit_reserve(3).expect("replace reserve");
1282 assert!(matches!(
1283 client.admit(true, false).await,
1284 Err(Error::RateLimited)
1285 ));
1286 let independent = reserved_client();
1287 independent
1288 .admit(true, false)
1289 .await
1290 .expect("independent budget");
1291 client.admit(true, true).await.expect("first exit");
1292 client.admit(true, true).await.expect("second exit");
1293 tokio::time::advance(Duration::from_millis(100)).await;
1294 for _ in 0..115 {
1295 client
1296 .admit(true, false)
1297 .await
1298 .expect("new ordinary capacity");
1299 }
1300 assert!(matches!(
1301 client.admit(true, false).await,
1302 Err(Error::RateLimited)
1303 ));
1304 tokio::time::advance(Duration::from_millis(900)).await;
1305 client.admit(true, false).await.expect("first exit expired");
1306 client
1307 .admit(true, false)
1308 .await
1309 .expect("second exit expired");
1310 assert!(matches!(
1311 client.admit(true, false).await,
1312 Err(Error::RateLimited)
1313 ));
1314 }
1315
1316 #[test]
1317 fn runtime_setter_reports_poison_and_preserves_policy_and_admission_state() {
1318 let client = reserved_client();
1319 poison(&client.inner.admission.inner.rate);
1320 assert!(matches!(
1321 client.set_exit_reserve(0),
1322 Err(Error::InvalidInput(_))
1323 ));
1324 let rate = client
1325 .inner
1326 .admission
1327 .inner
1328 .rate
1329 .lock()
1330 .unwrap_or_else(std::sync::PoisonError::into_inner);
1331 assert_eq!(rate.exit_reserve, 2);
1332 assert_eq!(rate.admitted.len(), 0);
1333 }
1334
1335 #[tokio::test(start_paused = true)]
1336 async fn runtime_waiter_rechecks_each_replacement_including_release_then_growth() {
1337 let client = reserved_client();
1338 client.set_exit_reserve(119).expect("raise reserve");
1339 client.admit(false, false).await.expect("one ordinary slot");
1340 let read = client.admit(false, false);
1341 tokio::pin!(read);
1342 assert!(futures_util::poll!(read.as_mut()).is_pending());
1343 client.set_exit_reserve(0).expect("release reserve");
1344 client.set_exit_reserve(119).expect("raise reserve");
1345 assert!(futures_util::poll!(read.as_mut()).is_pending());
1346 client.set_exit_reserve(118).expect("replace reserve");
1347 assert!(matches!(
1348 futures_util::poll!(read.as_mut()),
1349 std::task::Poll::Ready(Ok(()))
1350 ));
1351 assert!(matches!(
1352 client.admit(true, false).await,
1353 Err(Error::RateLimited)
1354 ));
1355 }
1356}