1use crate::validation::Validate;
7use crate::{
8 AccountID, Admission, ClientRequestID, Environment, Error, GenericRejection, OperationError,
9 RequestID, Result, Supplied, admission::REQUESTS_PER_SECOND, models::AcceptDatetimeFormat,
10};
11use futures_util::StreamExt;
12use reqwest::{Method, StatusCode, Url};
13use serde::{Serialize, de::DeserializeOwned};
14use serde_json::Value;
15use std::{
16 collections::HashSet,
17 fmt,
18 sync::{Arc, Mutex},
19 time::Duration,
20};
21use zeroize::Zeroizing;
22
23const DEFAULT_MAX_BODY: usize = 8 * 1024 * 1024;
24const DEFAULT_COOLDOWN: Duration = Duration::from_secs(1);
25const MAX_COOLDOWN: Duration = Duration::from_mins(1);
26
27#[derive(Debug, Clone)]
29pub struct ApiResponse<T> {
30 pub body: T,
32 pub request_id: Supplied<RequestID>,
36 pub next_page: Supplied<Url>,
43}
44
45#[derive(Clone)]
47pub struct Client {
48 pub(crate) inner: Arc<Inner>,
49}
50
51impl fmt::Debug for Client {
52 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
53 f.debug_struct("Client")
54 .field("environment", &self.inner.environment)
55 .finish_non_exhaustive()
56 }
57}
58
59pub(crate) struct Inner {
60 pub environment: Environment,
61 pub rest: Url,
62 pub stream: Url,
63 pub http: reqwest::Client,
64 pub stream_http: reqwest::Client,
65 pub token: Zeroizing<String>,
66 pub max_body: usize,
67 pub datetime_format: AcceptDatetimeFormat,
68 pub(crate) admission: Admission,
69 fenced: Mutex<HashSet<AccountID>>,
70 in_flight: Mutex<HashSet<AccountID>>,
71}
72
73#[derive(Clone, Copy)]
74pub(crate) enum MutationAdmission<'a> {
75 Ordinary(&'a AccountID),
76 Exit(&'a AccountID),
77}
78
79impl<'a> MutationAdmission<'a> {
80 fn account(self) -> &'a AccountID {
81 match self {
82 Self::Ordinary(account) | Self::Exit(account) => account,
83 }
84 }
85}
86
87pub(crate) struct RequestParts<'a> {
89 method: Method,
90 path: &'a str,
91 query: Option<Value>,
92 body: Option<Value>,
93 mutation: Option<MutationAdmission<'a>>,
94 client_request_id: Option<&'a ClientRequestID>,
95}
96
97impl<'a> RequestParts<'a> {
98 pub(crate) fn new(method: Method, path: &'a str) -> Self {
99 Self {
100 method,
101 path,
102 query: None,
103 body: None,
104 mutation: None,
105 client_request_id: None,
106 }
107 }
108 pub(crate) fn query<Q: Validate + Serialize>(mut self, query: &Q) -> Result<Self> {
109 query.validate()?;
110 self.query = Some(
111 serde_json::to_value(query).map_err(|error| Error::InvalidInput(error.to_string()))?,
112 );
113 Ok(self)
114 }
115 pub(crate) fn body<B: Validate + Serialize>(mut self, body: &B) -> Result<Self> {
116 body.validate()?;
117 self.body = Some(
118 serde_json::to_value(body).map_err(|error| Error::InvalidInput(error.to_string()))?,
119 );
120 Ok(self)
121 }
122 pub(crate) fn mutation(mut self, mutation: MutationAdmission<'a>) -> Self {
123 self.mutation = Some(mutation);
124 self
125 }
126 pub(crate) fn client_request_id(mut self, id: Option<&'a ClientRequestID>) -> Self {
127 self.client_request_id = id;
128 self
129 }
130}
131
132pub struct ClientBuilder {
134 environment: Environment,
135 token: Zeroizing<String>,
136 rest_override: Option<Url>,
137 stream_override: Option<Url>,
138 max_body: usize,
139 timeout: Duration,
140 exit_reserve: Option<u32>,
141 admission: Option<Admission>,
142 datetime_format: AcceptDatetimeFormat,
143}
144
145impl fmt::Debug for ClientBuilder {
146 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
147 f.debug_struct("ClientBuilder")
148 .field("environment", &self.environment)
149 .field("max_body", &self.max_body)
150 .finish_non_exhaustive()
151 }
152}
153
154impl Client {
155 pub fn set_exit_reserve(&self, requests: u32) -> Result<()> {
167 self.inner.admission.set_exit_reserve(requests)
168 }
169
170 pub fn builder(environment: Environment, bearer_token: impl Into<String>) -> ClientBuilder {
172 ClientBuilder {
173 environment,
174 token: Zeroizing::new(bearer_token.into()),
175 rest_override: None,
176 stream_override: None,
177 max_body: DEFAULT_MAX_BODY,
178 timeout: Duration::from_secs(30),
179 exit_reserve: None,
180 admission: None,
181 datetime_format: AcceptDatetimeFormat::Rfc3339,
182 }
183 }
184
185 pub fn acknowledge_reconciliation(&self, account: &AccountID) -> Result<()> {
194 let mut fenced = self
195 .inner
196 .fenced
197 .lock()
198 .map_err(|_| Error::Invariant("reconciliation lock poisoned"))?;
199 fenced.remove(account);
200 Ok(())
201 }
202
203 #[allow(clippy::too_many_lines)]
205 pub(crate) async fn execute<T: DeserializeOwned, R: DeserializeOwned>(
206 &self,
207 parts: RequestParts<'_>,
208 ) -> std::result::Result<ApiResponse<T>, OperationError<R>> {
209 let mutation_account = parts.mutation.map(MutationAdmission::account);
210 let exit = matches!(parts.mutation, Some(MutationAdmission::Exit(_)));
211 let url = Self::build_url(&self.inner.rest, parts.path, parts.query.as_ref())?;
212 let mut request = self
213 .inner
214 .http
215 .request(parts.method, url)
216 .bearer_auth(self.inner.token.as_str())
217 .header(
218 "Accept-Datetime-Format",
219 self.inner.datetime_format.as_str(),
220 );
221 if let Some(id) = parts.client_request_id {
222 request = request.header("ClientRequestID", id.as_str());
223 }
224 if let Some(body) = parts.body {
225 request = request.json(&body);
226 }
227 let request = request
229 .build()
230 .map_err(|error| Error::InvalidInput(error.to_string()))?;
231 self.admit(mutation_account.is_some(), exit).await?;
232 let mut guard = mutation_account
235 .map(|account| MutationGuard::new(self.inner.clone(), account))
236 .transpose()?;
237 let response =
238 self.inner
239 .http
240 .execute(request)
241 .await
242 .map_err(|e| match mutation_account {
243 Some(account) => Error::AmbiguousMutation {
244 account: account.clone(),
245 },
246 None => Error::Transport(e.to_string()),
247 })?;
248 let status = response.status();
249 let request_id = supplied_header(response.headers(), "RequestID");
250 let next_page = if status.is_success() {
251 parse_next_page(&self.inner.rest, response.headers())
252 } else {
253 Ok(Supplied::Absent)
254 };
255 if status == StatusCode::TOO_MANY_REQUESTS {
256 if let Some(ref mut guard) = guard {
258 guard.disarm();
259 }
260 self.install_cooldown(response.headers().get(reqwest::header::RETRY_AFTER))
261 .map_err(|source| Error::ProviderRateLimitedInvariant {
262 source: Box::new(source),
263 })?;
264 return Err(Error::ProviderRateLimited.into());
265 }
266 let definitive_rejection = matches!(status.as_u16(), 400 | 401 | 403 | 404 | 405);
267 if definitive_rejection && let Some(ref mut guard) = guard {
268 guard.disarm();
269 }
270 let bytes = read_bounded(response, self.inner.max_body)
271 .await
272 .map_err(|error| {
273 if definitive_rejection {
274 OperationError::RejectedUnreadable {
275 status: status.as_u16(),
276 request_id: request_id.clone(),
277 source: error,
278 }
279 } else if let Some(account) = mutation_account {
280 OperationError::Client(Error::AmbiguousMutation {
281 account: account.clone(),
282 })
283 } else {
284 OperationError::Client(error)
285 }
286 })?;
287 if status.is_success() {
288 let value = serde_json::from_slice(&bytes).map_err(|e| match mutation_account {
289 Some(account) => Error::AmbiguousMutation {
290 account: account.clone(),
291 },
292 None => Error::Decode(e.to_string()),
293 })?;
294 if let Some(ref mut guard) = guard {
295 guard.disarm();
296 }
297 return Ok(ApiResponse {
298 body: value,
299 request_id,
300 next_page: if mutation_account.is_some() {
303 next_page.unwrap_or(Supplied::Undecoded)
304 } else {
305 next_page?
306 },
307 });
308 }
309 if let Some(account) = mutation_account
310 && !definitive_rejection
311 {
312 return Err(Error::AmbiguousMutation {
313 account: account.clone(),
314 }
315 .into());
316 }
317 let body: Supplied<R> = if bytes.iter().all(u8::is_ascii_whitespace) {
318 Supplied::Absent
319 } else {
320 match serde_json::from_slice::<R>(&bytes) {
321 Ok(value) => Supplied::Decoded(value),
322 Err(_) => Supplied::Undecoded,
323 }
324 };
325 let reason = ProviderError::read(&bytes);
331 if let Some(ref mut guard) = guard {
332 guard.disarm();
333 }
334 Err(OperationError::Rejected {
335 status: status.as_u16(),
336 code: reason.error_code,
337 message: reason.error_message,
338 body,
339 })
340 }
341
342 pub async fn fetch_page<T: DeserializeOwned>(
350 &self,
351 next_page: &Url,
352 ) -> std::result::Result<ApiResponse<T>, OperationError<GenericRejection>> {
353 if next_page.scheme() != self.inner.rest.scheme()
354 || next_page.host_str() != self.inner.rest.host_str()
355 || next_page.port_or_known_default() != self.inner.rest.port_or_known_default()
356 || !next_page.path().starts_with("/v3/")
357 || next_page.fragment().is_some()
358 || next_page.username() != ""
359 || next_page.password().is_some()
360 {
361 return Err(Error::InvalidInput(
362 "pagination URL is outside the OANDA REST origin".into(),
363 )
364 .into());
365 }
366 let path = if let Some(query) = next_page.query() {
367 format!("{}?{query}", next_page.path())
368 } else {
369 next_page.path().to_owned()
370 };
371 self.execute(RequestParts::new(Method::GET, &path)).await
372 }
373
374 pub(crate) async fn admit(&self, mutation: bool, exit: bool) -> Result<()> {
375 self.inner.admission.admit(mutation, exit).await
376 }
377
378 pub(crate) fn build_url(base: &Url, path: &str, query: Option<&Value>) -> Result<Url> {
379 if !path.starts_with("/v3/") {
380 return Err(Error::InvalidInput("path must start with /v3/".into()));
381 }
382 let mut url = base
383 .join(path)
384 .map_err(|e| Error::InvalidInput(e.to_string()))?;
385 if let Some(query) = query {
386 let object = query
387 .as_object()
388 .ok_or_else(|| Error::InvalidInput("query must be an object".into()))?;
389 let mut pairs = url.query_pairs_mut();
390 for (key, value) in object {
391 if value.is_null() {
392 continue;
393 }
394 if let Some(items) = value.as_array() {
395 let csv = items
396 .iter()
397 .map(query_atom)
398 .collect::<Result<Vec<_>>>()?
399 .join(",");
400 pairs.append_pair(key, &csv);
401 } else {
402 pairs.append_pair(key, &query_atom(value)?);
403 }
404 }
405 }
406 Ok(url)
407 }
408
409 fn install_cooldown(&self, retry_after: Option<&reqwest::header::HeaderValue>) -> Result<()> {
416 let cooldown = match retry_after {
417 None => DEFAULT_COOLDOWN,
418 Some(value) => parse_retry_after(value).unwrap_or(MAX_COOLDOWN),
419 };
420 self.inner.admission.install_cooldown(cooldown)
421 }
422}
423
424fn parse_retry_after(value: &reqwest::header::HeaderValue) -> Option<Duration> {
429 let text = value.to_str().ok()?.trim();
430 if let Ok(seconds) = text.parse::<u64>() {
431 return Some(Duration::from_secs(seconds).min(MAX_COOLDOWN));
432 }
433 let when = chrono::DateTime::parse_from_rfc2822(text).ok()?;
434 let seconds = u64::try_from(
435 (when.with_timezone(&chrono::Utc) - chrono::Utc::now())
436 .num_seconds()
437 .max(0),
438 )
439 .ok()?;
440 Some(Duration::from_secs(seconds).min(MAX_COOLDOWN))
441}
442
443impl ClientBuilder {
444 #[must_use]
446 pub fn max_response_bytes(mut self, value: usize) -> Self {
447 self.max_body = value;
448 self
449 }
450
451 #[must_use]
458 pub fn exit_reserve(mut self, requests: u32) -> Self {
459 self.exit_reserve = Some(requests);
460 self
461 }
462
463 #[must_use]
468 pub fn admission(mut self, owner: Admission) -> Self {
469 self.admission = Some(owner);
470 self
471 }
472
473 #[must_use]
475 pub fn timeout(mut self, value: Duration) -> Self {
476 self.timeout = value;
477 self
478 }
479 #[must_use]
481 pub fn datetime_format(mut self, value: AcceptDatetimeFormat) -> Self {
482 self.datetime_format = value;
483 self
484 }
485 #[must_use]
488 pub fn endpoints(mut self, rest: Url, stream: Url) -> Self {
489 self.rest_override = Some(rest);
490 self.stream_override = Some(stream);
491 self
492 }
493
494 pub fn build(self) -> Result<Client> {
501 if self.token.trim().is_empty() || self.token.chars().any(char::is_control) {
502 return Err(Error::InvalidInput("invalid bearer token".into()));
503 }
504 if self.max_body == 0 || self.timeout.is_zero() {
505 return Err(Error::InvalidInput("limits must be positive".into()));
506 }
507 if self
508 .exit_reserve
509 .is_some_and(|reserve| reserve >= REQUESTS_PER_SECOND)
510 {
511 return Err(Error::InvalidInput(
512 "exit reserve must be less than 120 requests".into(),
513 ));
514 }
515 let admission = match self.admission {
516 Some(owner) => {
517 if let Some(requested) = self.exit_reserve
518 && requested != owner.exit_reserve()?
519 {
520 return Err(Error::InvalidInput(
521 "exit reserve conflicts with shared admission policy".into(),
522 ));
523 }
524 owner
525 }
526 None => Admission::with_exit_reserve(self.exit_reserve.unwrap_or(0))?,
527 };
528 let (default_rest, default_stream) = self.environment.urls()?;
529 let rest = self.rest_override.unwrap_or_else(|| default_rest.clone());
530 let stream = self
531 .stream_override
532 .unwrap_or_else(|| default_stream.clone());
533 validate_url(&rest)?;
534 validate_url(&stream)?;
535 let paired_loopback = is_loopback(&rest) && is_loopback(&stream);
536 if !paired_loopback && (rest != default_rest || stream != default_stream) {
537 return Err(Error::InvalidInput(
538 "endpoints must use the selected OANDA environment pair or loopback fixtures"
539 .into(),
540 ));
541 }
542 if matches!(self.datetime_format, AcceptDatetimeFormat::Unknown(_)) {
543 return Err(Error::InvalidInput("unsupported datetime format".into()));
544 }
545 let http = reqwest::Client::builder()
546 .connector_layer(admission.connection_layer())
547 .timeout(self.timeout)
548 .redirect(reqwest::redirect::Policy::none())
549 .retry(reqwest::retry::never())
550 .no_proxy()
551 .build()
552 .map_err(|e| Error::InvalidInput(e.to_string()))?;
553 let stream_http = stream_transport(reqwest::Client::builder(), self.timeout)
554 .connector_layer(admission.connection_layer())
555 .build()
556 .map_err(|e| Error::InvalidInput(e.to_string()))?;
557 Ok(Client {
558 inner: Arc::new(Inner {
559 environment: self.environment,
560 rest,
561 stream,
562 http,
563 stream_http,
564 token: self.token,
565 max_body: self.max_body,
566 datetime_format: self.datetime_format,
567 admission,
568 fenced: Mutex::new(HashSet::new()),
569 in_flight: Mutex::new(HashSet::new()),
570 }),
571 })
572 }
573}
574
575fn validate_url(url: &Url) -> Result<()> {
576 let host = url
577 .host_str()
578 .ok_or_else(|| Error::InvalidInput("endpoint must have a host".into()))?;
579 let loopback = host == "localhost" || host == "127.0.0.1" || host == "::1";
580 if url.scheme() != "https" && !(loopback && url.scheme() == "http") {
581 return Err(Error::InvalidInput("remote endpoints require HTTPS".into()));
582 }
583 if url.path() != "/"
584 || url.username() != ""
585 || url.password().is_some()
586 || url.query().is_some()
587 || url.fragment().is_some()
588 {
589 return Err(Error::InvalidInput(
590 "endpoint URL must not contain credentials, query, or fragment".into(),
591 ));
592 }
593 Ok(())
594}
595
596fn is_loopback(url: &Url) -> bool {
597 matches!(url.host_str(), Some("localhost" | "127.0.0.1" | "::1"))
598}
599
600fn query_atom(value: &serde_json::Value) -> Result<String> {
601 match value {
602 serde_json::Value::String(s) => Ok(s.clone()),
603 serde_json::Value::Number(n) => Ok(n.to_string()),
604 serde_json::Value::Bool(b) => Ok(b.to_string()),
605 _ => Err(Error::InvalidInput("unsupported query value".into())),
606 }
607}
608
609fn supplied_header(headers: &reqwest::header::HeaderMap, name: &str) -> Supplied<RequestID> {
615 match headers.get(name) {
616 None => Supplied::Absent,
617 Some(value) => match value.to_str() {
618 Ok(text) => match RequestID::new(text) {
619 Ok(id) => Supplied::Decoded(id),
620 Err(_) => Supplied::Undecoded,
621 },
622 Err(_) => Supplied::Undecoded,
623 },
624 }
625}
626
627fn parse_next_page(base: &Url, headers: &reqwest::header::HeaderMap) -> Result<Supplied<Url>> {
636 let Some(link) = headers.get(reqwest::header::LINK) else {
637 return Ok(Supplied::Absent);
638 };
639 let Ok(value) = link.to_str() else {
640 return Ok(Supplied::Undecoded);
641 };
642 let Some(item) = value
643 .split(',')
644 .find(|item| item.contains("rel=\"next\"") || item.contains("rel=next"))
645 else {
646 return Ok(Supplied::Absent);
647 };
648 let Some(start) = item.find('<') else {
654 return Err(Error::Decode("invalid pagination link".into()));
655 };
656 let Some(end) = item[start + 1..].find('>') else {
657 return Err(Error::Decode("invalid pagination link".into()));
658 };
659 let url = base
660 .join(&item[start + 1..start + 1 + end])
661 .map_err(|e| Error::Decode(e.to_string()))?;
662 if url.scheme() != base.scheme()
663 || url.host_str() != base.host_str()
664 || url.port_or_known_default() != base.port_or_known_default()
665 || !url.path().starts_with("/v3/")
666 {
667 return Err(Error::Decode(
668 "pagination link changed origin or API version".into(),
669 ));
670 }
671 Ok(Supplied::Decoded(url))
672}
673
674pub(crate) fn segment(value: &str) -> String {
675 const PATH_SEGMENT: &percent_encoding::AsciiSet = &percent_encoding::NON_ALPHANUMERIC
676 .remove(b'-')
677 .remove(b'_')
678 .remove(b'.')
679 .remove(b'~');
680 percent_encoding::utf8_percent_encode(value, PATH_SEGMENT).to_string()
681}
682
683pub(crate) async fn read_bounded(response: reqwest::Response, max: usize) -> Result<Vec<u8>> {
684 let mut stream = response.bytes_stream();
685 let mut bytes = Vec::new();
686 while let Some(next) = stream.next().await {
687 let chunk = next.map_err(|e| Error::Transport(e.to_string()))?;
688 if bytes.len().saturating_add(chunk.len()) > max {
689 return Err(Error::ResponseTooLarge);
690 }
691 bytes.extend_from_slice(&chunk);
692 }
693 Ok(bytes)
694}
695
696#[derive(serde::Deserialize)]
697#[serde(rename_all = "camelCase")]
698struct ProviderError {
699 error_code: Option<String>,
700 error_message: Option<String>,
701}
702
703impl ProviderError {
704 fn read(bytes: &[u8]) -> Self {
708 let value: serde_json::Value = match serde_json::from_slice(bytes) {
709 Ok(value) => value,
710 Err(_) => return Self::UNREAD,
711 };
712 Self {
713 error_code: value
714 .get("errorCode")
715 .and_then(serde_json::Value::as_str)
716 .map(str::to_owned),
717 error_message: value
718 .get("errorMessage")
719 .and_then(serde_json::Value::as_str)
720 .map(str::to_owned),
721 }
722 }
723
724 const UNREAD: Self = Self {
728 error_code: None,
729 error_message: None,
730 };
731}
732
733struct MutationGuard {
734 inner: Arc<Inner>,
735 account: AccountID,
736 armed: bool,
737}
738impl MutationGuard {
739 fn new(inner: Arc<Inner>, account: &AccountID) -> Result<Self> {
740 let mut in_flight = inner
743 .in_flight
744 .lock()
745 .map_err(|_| Error::Invariant("mutation lock poisoned"))?;
746 if inner
747 .fenced
748 .lock()
749 .map_err(|_| Error::Invariant("reconciliation lock poisoned"))?
750 .contains(account)
751 {
752 return Err(Error::ReconciliationRequired {
753 account: account.clone(),
754 });
755 }
756 if !in_flight.insert(account.clone()) {
757 return Err(Error::MutationInFlight {
758 account: account.clone(),
759 });
760 }
761 drop(in_flight);
762 Ok(Self {
763 inner,
764 account: account.clone(),
765 armed: true,
766 })
767 }
768 fn disarm(&mut self) {
769 self.armed = false;
770 }
771}
772impl Drop for MutationGuard {
773 fn drop(&mut self) {
774 if self.armed {
779 let mut fenced = self
780 .inner
781 .fenced
782 .lock()
783 .unwrap_or_else(std::sync::PoisonError::into_inner);
784 fenced.insert(self.account.clone());
785 }
786 let mut in_flight = self
787 .inner
788 .in_flight
789 .lock()
790 .unwrap_or_else(std::sync::PoisonError::into_inner);
791 in_flight.remove(&self.account);
792 }
793}
794
795fn stream_transport(builder: reqwest::ClientBuilder, timeout: Duration) -> reqwest::ClientBuilder {
796 builder
797 .http1_only()
798 .connect_timeout(timeout)
799 .redirect(reqwest::redirect::Policy::none())
800 .retry(reqwest::retry::never())
801 .no_proxy()
802}
803
804#[cfg(test)]
805#[allow(clippy::expect_used, clippy::panic)]
808mod tests {
809 use super::*;
810 use tokio::time::Instant as RateInstant;
811
812 #[tokio::test]
813 async fn streaming_uses_http11_when_the_caller_enables_http2() {
814 use tokio::io::{AsyncReadExt, AsyncWriteExt};
815 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
816 .await
817 .expect("bind");
818 let address = listener.local_addr().expect("address");
819 let server = tokio::spawn(async move {
820 let (mut socket, _) = listener.accept().await.expect("accept");
821 let mut bytes = Vec::new();
822 let mut buffer = [0; 1024];
823 loop {
824 let count = socket.read(&mut buffer).await.expect("read request");
825 if count == 0 {
826 break;
827 }
828 bytes.extend_from_slice(&buffer[..count]);
829 if bytes.windows(4).any(|window| window == b"\r\n\r\n") {
830 break;
831 }
832 }
833 socket
834 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
835 .await
836 .expect("response");
837 bytes
838 });
839 let client = stream_transport(
840 reqwest::Client::builder().http2_prior_knowledge(),
841 Duration::from_secs(5),
842 )
843 .build()
844 .expect("client");
845 let response = client
846 .get(format!(
847 "http://{address}/v3/accounts/fixture/transactions/stream"
848 ))
849 .send()
850 .await;
851 let request = server.await.expect("server");
852 assert!(
853 request.starts_with(b"GET /v3/accounts/fixture/transactions/stream HTTP/1.1\r\n"),
854 "request protocol: {}",
855 String::from_utf8_lossy(&request)
856 );
857 assert_eq!(
858 response.expect("stream response").version(),
859 reqwest::Version::HTTP_11
860 );
861 }
862
863 #[tokio::test]
864 async fn mutation_rate_admission_is_shared_by_clones() {
865 let client = Client::builder(Environment::Practice, "fixture").build();
866 assert!(client.is_ok());
867 let Ok(client) = client else { return };
868 if let Ok(mut rate) = client.inner.admission.inner.rate.lock() {
869 for _ in 0..REQUESTS_PER_SECOND {
870 rate.admitted.push_back(RateInstant::now());
871 }
872 }
873 assert!(matches!(
874 client.clone().admit(true, false).await,
875 Err(Error::RateLimited)
876 ));
877 }
878
879 #[test]
880 fn account_reservation_and_reconciliation_are_shared() {
881 let client = Client::builder(Environment::Practice, "fixture").build();
882 assert!(client.is_ok());
883 let Ok(client) = client else { return };
884 let account = AccountID::new("101-001-1-001");
885 assert!(account.is_ok());
886 let Ok(account) = account else { return };
887 let first = MutationGuard::new(client.inner.clone(), &account);
888 assert!(first.is_ok());
889 let Ok(first) = first else { return };
890 assert!(matches!(
891 MutationGuard::new(client.clone().inner, &account),
892 Err(Error::MutationInFlight { .. })
893 ));
894 drop(first);
895 assert!(matches!(
896 MutationGuard::new(client.inner.clone(), &account),
897 Err(Error::ReconciliationRequired { .. })
898 ));
899 let other = AccountID::new("101-001-1-002");
900 assert!(other.is_ok());
901 if let Ok(other) = other {
902 let mut independent = MutationGuard::new(client.inner.clone(), &other);
903 assert!(independent.is_ok());
904 if let Ok(ref mut guard) = independent {
905 guard.disarm();
906 }
907 }
908 client
909 .acknowledge_reconciliation(&account)
910 .expect("fence released");
911 let mut next = MutationGuard::new(client.inner.clone(), &account);
912 assert!(next.is_ok());
913 if let Ok(ref mut guard) = next {
914 guard.disarm();
915 }
916 }
917
918 fn poison<T>(lock: &Mutex<T>) {
919 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
920 let Ok(_held) = lock.lock() else {
921 return;
922 };
923 panic!("poison the lock");
924 }));
925 assert!(result.is_err());
926 assert!(lock.is_poisoned());
927 }
928
929 #[test]
930 fn an_ambiguous_mutation_is_fenced_even_after_a_poisoned_lock() {
931 let client = Client::builder(Environment::Practice, "fixture").build();
932 assert!(client.is_ok());
933 let Ok(client) = client else { return };
934 let account = AccountID::new("101-001-1-001");
935 assert!(account.is_ok());
936 let Ok(account) = account else { return };
937 let guard = MutationGuard::new(client.inner.clone(), &account);
938 assert!(guard.is_ok());
939 let Ok(guard) = guard else { return };
940 poison(&client.inner.fenced);
943 poison(&client.inner.in_flight);
944 drop(guard);
945 let fenced = client
946 .inner
947 .fenced
948 .lock()
949 .unwrap_or_else(std::sync::PoisonError::into_inner);
950 assert!(
951 fenced.contains(&account),
952 "the ambiguity must be fenced, not dropped"
953 );
954 let in_flight = client
955 .inner
956 .in_flight
957 .lock()
958 .unwrap_or_else(std::sync::PoisonError::into_inner);
959 assert!(!in_flight.contains(&account));
960 }
961
962 #[test]
963 fn a_failed_acknowledgement_is_reported_and_leaves_the_account_fenced() {
964 let client = Client::builder(Environment::Practice, "fixture").build();
965 assert!(client.is_ok());
966 let Ok(client) = client else { return };
967 let account = AccountID::new("101-001-1-001");
968 assert!(account.is_ok());
969 let Ok(account) = account else { return };
970 {
971 let mut fenced = client
972 .inner
973 .fenced
974 .lock()
975 .unwrap_or_else(std::sync::PoisonError::into_inner);
976 fenced.insert(account.clone());
977 }
978 poison(&client.inner.fenced);
979 assert!(matches!(
980 client.acknowledge_reconciliation(&account),
981 Err(Error::Invariant(_))
982 ));
983 let fenced = client
984 .inner
985 .fenced
986 .lock()
987 .unwrap_or_else(std::sync::PoisonError::into_inner);
988 assert!(fenced.contains(&account));
989 }
990
991 fn assert_cooldown(header: Option<&str>, expected: Duration) {
992 let client = Client::builder(Environment::Practice, "fixture").build();
993 assert!(client.is_ok());
994 let Ok(client) = client else { return };
995 let header =
996 header.map(|text| reqwest::header::HeaderValue::from_str(text).expect("header"));
997 let start = RateInstant::now();
998 client
999 .install_cooldown(header.as_ref())
1000 .expect("install cooldown");
1001 let rate = client
1002 .inner
1003 .admission
1004 .inner
1005 .rate
1006 .lock()
1007 .unwrap_or_else(std::sync::PoisonError::into_inner);
1008 let until = rate.cooldown_until.expect("cooldown installed");
1009 let tolerance = Duration::from_secs(1);
1014 assert!(
1015 until + tolerance >= start + expected,
1016 "cooldown shorter than {expected:?}"
1017 );
1018 assert!(
1019 until <= RateInstant::now() + expected + tolerance,
1020 "cooldown longer than {expected:?}"
1021 );
1022 }
1023
1024 #[test]
1025 fn retry_after_reads_both_rfc_9110_forms() {
1026 assert_cooldown(Some("7"), Duration::from_secs(7));
1027 let later = chrono::Utc::now() + chrono::Duration::seconds(30);
1029 assert_cooldown(
1030 Some(&later.format("%a, %d %b %Y %H:%M:%S GMT").to_string()),
1031 Duration::from_secs(29),
1032 );
1033 let earlier = chrono::Utc::now() - chrono::Duration::seconds(30);
1035 assert_cooldown(
1036 Some(&earlier.format("%a, %d %b %Y %H:%M:%S GMT").to_string()),
1037 Duration::ZERO,
1038 );
1039 assert_cooldown(Some("9999"), MAX_COOLDOWN);
1040 }
1041
1042 #[test]
1043 fn an_unreadable_or_absent_retry_after_is_not_read_as_a_provider_instruction() {
1044 assert_cooldown(None, DEFAULT_COOLDOWN);
1046 assert_cooldown(Some("soon"), MAX_COOLDOWN);
1049 }
1050
1051 fn reserved_client() -> Client {
1052 Client::builder(Environment::Practice, "fixture")
1053 .exit_reserve(2)
1054 .build()
1055 .expect("valid reserve")
1056 }
1057
1058 #[test]
1059 fn exit_reserve_rejects_the_full_window_and_larger_values() {
1060 for reserve in [120, 121, u32::MAX] {
1061 assert!(matches!(
1062 Client::builder(Environment::Practice, "fixture")
1063 .exit_reserve(reserve)
1064 .build(),
1065 Err(Error::InvalidInput(_))
1066 ));
1067 }
1068 }
1069
1070 #[tokio::test(start_paused = true)]
1071 async fn ordinary_admission_leaves_two_exits_and_preserves_the_aggregate_limit() {
1072 let client = reserved_client();
1073 for _ in 0..118 {
1074 client.admit(false, false).await.expect("ordinary slot");
1075 }
1076 assert!(matches!(
1077 client.admit(true, false).await,
1078 Err(Error::RateLimited)
1079 ));
1080 let read = client.admit(false, false);
1081 tokio::pin!(read);
1082 assert!(futures_util::poll!(read.as_mut()).is_pending());
1083 client.clone().admit(true, true).await.expect("first exit");
1084 client.clone().admit(true, true).await.expect("second exit");
1085 assert!(matches!(
1086 client.admit(true, true).await,
1087 Err(Error::RateLimited)
1088 ));
1089 assert!(futures_util::poll!(read.as_mut()).is_pending());
1090 tokio::time::advance(Duration::from_millis(999)).await;
1091 assert!(futures_util::poll!(read.as_mut()).is_pending());
1092 tokio::time::advance(Duration::from_millis(1)).await;
1093 read.await.expect("ordinary slot after exact expiry");
1094 }
1095
1096 #[tokio::test(start_paused = true)]
1097 async fn concurrent_clones_cannot_exceed_the_two_remaining_exit_slots() {
1098 let client = reserved_client();
1099 for _ in 0..118 {
1100 client.admit(false, false).await.expect("ordinary slot");
1101 }
1102 let mut tasks = Vec::new();
1103 for _ in 0..20 {
1104 let clone = client.clone();
1105 tasks.push(tokio::spawn(async move { clone.admit(true, true).await }));
1106 }
1107 let mut admitted = 0;
1108 let mut refused = 0;
1109 for task in tasks {
1110 match task.await.expect("join") {
1111 Ok(()) => admitted += 1,
1112 Err(Error::RateLimited) => refused += 1,
1113 result => panic!("unexpected admission: {result:?}"),
1114 }
1115 }
1116 assert_eq!(admitted, 2);
1117 assert_eq!(refused, 18);
1118 }
1119
1120 #[tokio::test(start_paused = true)]
1121 async fn cancelled_ordinary_waiters_do_not_charge_the_next_window() {
1122 let client = reserved_client();
1123 for _ in 0..118 {
1124 client.admit(false, false).await.expect("ordinary slot");
1125 }
1126 {
1127 let read = client.admit(false, false);
1128 tokio::pin!(read);
1129 assert!(futures_util::poll!(read.as_mut()).is_pending());
1130 }
1131 tokio::time::advance(Duration::from_secs(1)).await;
1132 for _ in 0..118 {
1133 client.admit(true, false).await.expect("new ordinary slot");
1134 }
1135 assert!(matches!(
1136 client.admit(true, false).await,
1137 Err(Error::RateLimited)
1138 ));
1139 client.admit(true, true).await.expect("first exit");
1140 client.admit(true, true).await.expect("second exit");
1141 }
1142
1143 #[tokio::test(start_paused = true)]
1144 async fn provider_cooldown_blocks_both_classes_until_exact_expiry() {
1145 let client = reserved_client();
1146 client
1147 .install_cooldown(Some(&reqwest::header::HeaderValue::from_static("2")))
1148 .expect("install cooldown");
1149 assert!(matches!(
1150 client.clone().admit(true, false).await,
1151 Err(Error::RateLimited)
1152 ));
1153 assert!(matches!(
1154 client.clone().admit(true, true).await,
1155 Err(Error::RateLimited)
1156 ));
1157 let read = client.admit(false, false);
1158 tokio::pin!(read);
1159 assert!(futures_util::poll!(read.as_mut()).is_pending());
1160 tokio::time::advance(Duration::from_millis(1999)).await;
1161 assert!(futures_util::poll!(read.as_mut()).is_pending());
1162 assert!(matches!(
1163 client.admit(true, true).await,
1164 Err(Error::RateLimited)
1165 ));
1166 tokio::time::advance(Duration::from_millis(1)).await;
1167 read.await.expect("read after cooldown");
1168 client.admit(true, true).await.expect("exit after cooldown");
1169 }
1170
1171 #[tokio::test(start_paused = true)]
1172 async fn a_shorter_provider_cooldown_cannot_release_an_existing_deadline_early() {
1173 let client = reserved_client();
1174 client
1175 .install_cooldown(Some(&reqwest::header::HeaderValue::from_static("5")))
1176 .expect("install cooldown");
1177 tokio::time::advance(Duration::from_secs(1)).await;
1178 client
1179 .install_cooldown(Some(&reqwest::header::HeaderValue::from_static("1")))
1180 .expect("install cooldown");
1181 tokio::time::advance(Duration::from_secs(1)).await;
1182 assert!(matches!(
1183 client.admit(true, true).await,
1184 Err(Error::RateLimited)
1185 ));
1186 assert!(matches!(
1187 client.admit(true, false).await,
1188 Err(Error::RateLimited)
1189 ));
1190 tokio::time::advance(Duration::from_secs(3)).await;
1191 client
1192 .admit(true, true)
1193 .await
1194 .expect("original cooldown expired");
1195 }
1196
1197 #[tokio::test(start_paused = true)]
1198 async fn runtime_release_wakes_all_registered_waiters_without_advancing_time() {
1199 let client = reserved_client();
1200 for _ in 0..118 {
1201 client.admit(false, false).await.expect("ordinary slot");
1202 }
1203 let first = client.admit(false, false);
1204 let second = client.admit(false, false);
1205 tokio::pin!(first, second);
1206 assert!(futures_util::poll!(first.as_mut()).is_pending());
1207 assert!(futures_util::poll!(second.as_mut()).is_pending());
1208 let started = RateInstant::now();
1209 client.clone().set_exit_reserve(0).expect("release reserve");
1210 assert!(matches!(
1211 futures_util::poll!(first.as_mut()),
1212 std::task::Poll::Ready(Ok(()))
1213 ));
1214 assert!(matches!(
1215 futures_util::poll!(second.as_mut()),
1216 std::task::Poll::Ready(Ok(()))
1217 ));
1218 assert_eq!(RateInstant::now(), started);
1219 assert!(matches!(
1220 client.admit(true, true).await,
1221 Err(Error::RateLimited)
1222 ));
1223 }
1224
1225 #[tokio::test(start_paused = true)]
1226 async fn runtime_growth_rechecks_waiters_and_preserves_spent_capacity() {
1227 let client = reserved_client();
1228 for _ in 0..118 {
1229 client.admit(false, false).await.expect("ordinary slot");
1230 }
1231 let read = client.admit(false, false);
1232 tokio::pin!(read);
1233 assert!(futures_util::poll!(read.as_mut()).is_pending());
1234 client.set_exit_reserve(0).expect("release reserve");
1235 client
1236 .clone()
1237 .set_exit_reserve(119)
1238 .expect("replace reserve");
1239 assert!(futures_util::poll!(read.as_mut()).is_pending());
1240 client.admit(true, true).await.expect("first exit");
1241 client.admit(true, true).await.expect("second exit");
1242 assert!(matches!(
1243 client.admit(true, true).await,
1244 Err(Error::RateLimited)
1245 ));
1246 client
1247 .set_exit_reserve(0)
1248 .expect("release exhausted reserve");
1249 client
1250 .set_exit_reserve(119)
1251 .expect("raise exhausted reserve");
1252 assert!(matches!(
1253 client.admit(true, true).await,
1254 Err(Error::RateLimited)
1255 ));
1256 client.set_exit_reserve(0).expect("release reserve");
1257 assert!(futures_util::poll!(read.as_mut()).is_pending());
1258 tokio::time::advance(Duration::from_secs(1)).await;
1259 assert!(matches!(
1260 futures_util::poll!(read.as_mut()),
1261 std::task::Poll::Ready(Ok(()))
1262 ));
1263 }
1264
1265 #[tokio::test(start_paused = true)]
1266 async fn runtime_changes_preserve_cooldown_window_and_invalid_policy_state() {
1267 let client = reserved_client();
1268 for _ in 0..118 {
1269 client.admit(false, false).await.expect("ordinary slot");
1270 }
1271 client
1272 .install_cooldown(Some(&reqwest::header::HeaderValue::from_static("2")))
1273 .expect("install cooldown");
1274 for reserve in [119, 0, 2] {
1275 client.set_exit_reserve(reserve).expect("replace reserve");
1276 assert!(matches!(
1277 client.admit(true, true).await,
1278 Err(Error::RateLimited)
1279 ));
1280 }
1281 for reserve in [120, u32::MAX] {
1282 assert!(matches!(
1283 client.set_exit_reserve(reserve),
1284 Err(Error::InvalidInput(_))
1285 ));
1286 let rate = client.inner.admission.inner.rate.lock().expect("rate lock");
1287 assert_eq!(rate.exit_reserve, 2);
1288 assert_eq!(rate.admitted.len(), 118);
1289 }
1290 client.set_exit_reserve(0).expect("release reserve");
1291 let read = client.admit(false, false);
1292 tokio::pin!(read);
1293 assert!(futures_util::poll!(read.as_mut()).is_pending());
1294 tokio::time::advance(Duration::from_secs(1)).await;
1295 assert!(futures_util::poll!(read.as_mut()).is_pending());
1296 tokio::time::advance(Duration::from_secs(1)).await;
1297 assert!(matches!(
1298 futures_util::poll!(read.as_mut()),
1299 std::task::Poll::Ready(Ok(()))
1300 ));
1301 for _ in 0..119 {
1302 client
1303 .admit(true, true)
1304 .await
1305 .expect("remaining aggregate capacity");
1306 }
1307 assert!(matches!(
1308 client.admit(true, true).await,
1309 Err(Error::RateLimited)
1310 ));
1311 }
1312
1313 #[tokio::test(start_paused = true)]
1314 async fn runtime_replacement_keeps_each_admission_deadline_and_independent_client() {
1315 let client = reserved_client();
1316 for _ in 0..118 {
1317 client.admit(false, false).await.expect("ordinary slot");
1318 }
1319 tokio::time::advance(Duration::from_millis(900)).await;
1320 client.set_exit_reserve(3).expect("replace reserve");
1321 assert!(matches!(
1322 client.admit(true, false).await,
1323 Err(Error::RateLimited)
1324 ));
1325 let independent = reserved_client();
1326 independent
1327 .admit(true, false)
1328 .await
1329 .expect("independent budget");
1330 client.admit(true, true).await.expect("first exit");
1331 client.admit(true, true).await.expect("second exit");
1332 tokio::time::advance(Duration::from_millis(100)).await;
1333 for _ in 0..115 {
1334 client
1335 .admit(true, false)
1336 .await
1337 .expect("new ordinary capacity");
1338 }
1339 assert!(matches!(
1340 client.admit(true, false).await,
1341 Err(Error::RateLimited)
1342 ));
1343 tokio::time::advance(Duration::from_millis(900)).await;
1344 client.admit(true, false).await.expect("first exit expired");
1345 client
1346 .admit(true, false)
1347 .await
1348 .expect("second exit expired");
1349 assert!(matches!(
1350 client.admit(true, false).await,
1351 Err(Error::RateLimited)
1352 ));
1353 }
1354
1355 #[test]
1356 fn runtime_setter_reports_poison_and_preserves_policy_and_admission_state() {
1357 let client = reserved_client();
1358 poison(&client.inner.admission.inner.rate);
1359 assert!(matches!(
1360 client.set_exit_reserve(0),
1361 Err(Error::Invariant(_))
1362 ));
1363 let rate = client
1364 .inner
1365 .admission
1366 .inner
1367 .rate
1368 .lock()
1369 .unwrap_or_else(std::sync::PoisonError::into_inner);
1370 assert_eq!(rate.exit_reserve, 2);
1371 assert_eq!(rate.admitted.len(), 0);
1372 }
1373
1374 #[tokio::test(start_paused = true)]
1375 async fn runtime_waiter_rechecks_each_replacement_including_release_then_growth() {
1376 let client = reserved_client();
1377 client.set_exit_reserve(119).expect("raise reserve");
1378 client.admit(false, false).await.expect("one ordinary slot");
1379 let read = client.admit(false, false);
1380 tokio::pin!(read);
1381 assert!(futures_util::poll!(read.as_mut()).is_pending());
1382 client.set_exit_reserve(0).expect("release reserve");
1383 client.set_exit_reserve(119).expect("raise reserve");
1384 assert!(futures_util::poll!(read.as_mut()).is_pending());
1385 client.set_exit_reserve(118).expect("replace reserve");
1386 assert!(matches!(
1387 futures_util::poll!(read.as_mut()),
1388 std::task::Poll::Ready(Ok(()))
1389 ));
1390 assert!(matches!(
1391 client.admit(true, false).await,
1392 Err(Error::RateLimited)
1393 ));
1394 }
1395 #[test]
1396 fn an_unreadable_fence_never_leaves_a_partial_reservation() {
1397 let client = Client::builder(Environment::Practice, "fixture")
1398 .build()
1399 .expect("client");
1400 let account = AccountID::new("fixture-account").expect("account");
1401 poison(&client.inner.fenced);
1402 assert!(matches!(
1403 MutationGuard::new(client.inner.clone(), &account),
1404 Err(Error::Invariant(_))
1405 ));
1406 assert!(
1407 client
1408 .inner
1409 .in_flight
1410 .lock()
1411 .expect("reservation lock")
1412 .is_empty()
1413 );
1414 }
1415
1416 #[tokio::test]
1417 async fn cooldown_failure_preserves_429_without_fencing_or_partial_policy_changes() {
1418 use tokio::io::AsyncWriteExt;
1419 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
1420 .await
1421 .expect("bind");
1422 let url = Url::parse(&format!(
1423 "http://{}/",
1424 listener.local_addr().expect("address")
1425 ))
1426 .expect("URL");
1427 let client = Client::builder(Environment::Practice, "fixture")
1428 .endpoints(url.clone(), url)
1429 .build()
1430 .expect("client");
1431 let admission = client.inner.admission.clone();
1432 let server = tokio::spawn(async move {
1433 let (mut socket, _) = listener.accept().await.expect("accept");
1434 crate::test_support::read_request(&mut socket, &mut Vec::new())
1435 .await
1436 .expect("request");
1437 poison(&admission.inner.rate);
1438 socket.write_all(b"HTTP/1.1 429 Too Many Requests\r\nRetry-After: 2\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await.expect("429");
1439 });
1440 let account = AccountID::new("fixture-account").expect("account");
1441 let result = client
1442 .configure_account(
1443 &account,
1444 &crate::accounts::ConfigureAccountBody {
1445 alias: Some("fixture".into()),
1446 margin_rate: None,
1447 },
1448 )
1449 .await;
1450 assert!(
1451 matches!(result, Err(OperationError::Client(Error::ProviderRateLimitedInvariant { source })) if matches!(*source, Error::Invariant(_)))
1452 );
1453 server.await.expect("server");
1454 assert!(client.inner.fenced.lock().expect("fence lock").is_empty());
1455 assert!(
1456 client
1457 .inner
1458 .in_flight
1459 .lock()
1460 .expect("reservation lock")
1461 .is_empty()
1462 );
1463 let rate = client
1464 .inner
1465 .admission
1466 .inner
1467 .rate
1468 .lock()
1469 .unwrap_or_else(std::sync::PoisonError::into_inner);
1470 assert_eq!(rate.admitted.len(), 1);
1471 assert!(
1472 rate.cooldown_until.is_none(),
1473 "failed installation must preserve policy"
1474 );
1475 }
1476}