#![doc = include_str!("client/README.md")]
use crate::{
LeaseId, Revision, Version,
record::{KeyWithMetadata, Metadata, Record},
};
use std::{ops::AsyncFn, sync::Arc};
mod builder;
pub use builder::{BuildError, ClientBuilder};
mod driver;
mod metrics;
pub use metrics::{MetricsCollector, RequestCount, RequestCounter};
#[derive(Clone)]
pub struct Client {
inner: Arc<ClientInner>,
}
mod construct;
mod ops;
pub use ops::*;
struct Credentials {
username: String,
password: String,
}
#[derive(Debug, Clone, PartialEq)]
pub enum RetryPolicy {
WithDeadline(std::time::Duration),
Never,
Forever,
}
impl Default for RetryPolicy {
fn default() -> Self {
Self::WithDeadline(std::time::Duration::from_secs(5))
}
}
impl From<std::time::Duration> for RetryPolicy {
fn from(d: std::time::Duration) -> Self {
Self::WithDeadline(d)
}
}
struct AuthState {
credentials: Option<Credentials>,
token: std::sync::RwLock<(u64, Option<tonic::metadata::AsciiMetadataValue>)>,
refresh: tokio::sync::Mutex<()>,
}
struct ClientInner {
channel: tonic::transport::Channel,
metrics: Option<Box<dyn MetricsCollector>>,
auth: Option<AuthState>,
retry_policy: RetryPolicy,
max_response_size: usize,
}
fn metadata_from_pb(r: &crate::pb::mvccpb::KeyValue) -> Metadata {
Metadata {
create_revision: Revision::new(r.create_revision).unwrap(),
modified_revision: Revision::new(r.mod_revision).unwrap(),
version: Version::new(r.version as u64),
lease: LeaseId::new(r.lease),
}
}
fn record_from_pb(r: crate::pb::mvccpb::KeyValue) -> Record {
let metadata = metadata_from_pb(&r);
Record::new(r.key, r.value, metadata)
}
fn key_with_metadata_from_pb(r: crate::pb::mvccpb::KeyValue) -> KeyWithMetadata {
let metadata = metadata_from_pb(&r);
KeyWithMetadata::new(r.key, metadata)
}
fn is_client_side_timeout(status: &tonic::Status) -> bool {
status.code() == tonic::Code::Cancelled && status.message() == "Timeout expired"
}
fn is_transport_error(status: &tonic::Status) -> bool {
use std::error::Error as _;
status.code() == tonic::Code::Unknown && status.source().is_some() && status.message() == "transport error"
}
fn is_connect_error(status: &tonic::Status) -> bool {
use std::error::Error as _;
let mut source = status.source();
while let Some(err) = source {
if err.is::<tonic::ConnectError>() {
return true;
}
source = err.source();
}
false
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(crate) enum Idempotency {
Immutable,
Mutable,
}
trait ServiceClient: Sized {
fn with_max_decoding_message_size(self, limit: usize) -> Self;
}
macro_rules! impl_service_client {
($($client:ty),+ $(,)?) => {$(
impl ServiceClient for $client {
fn with_max_decoding_message_size(self, limit: usize) -> Self {
self.max_decoding_message_size(limit)
}
}
)+};
}
impl_service_client!(
crate::pb::etcdserverpb::auth_client::AuthClient<tonic::transport::Channel>,
crate::pb::etcdserverpb::cluster_client::ClusterClient<tonic::transport::Channel>,
crate::pb::etcdserverpb::kv_client::KvClient<tonic::transport::Channel>,
crate::pb::etcdserverpb::lease_client::LeaseClient<tonic::transport::Channel>,
crate::pb::etcdserverpb::watch_client::WatchClient<tonic::transport::Channel>,
);
impl ClientInner {
fn service_client<C: ServiceClient>(&self, create: impl FnOnce(tonic::transport::Channel) -> C) -> C {
create(self.channel.clone()).with_max_decoding_message_size(self.max_response_size)
}
async fn wrap_unary_call<R, GrpcClient, Request>(
&self,
idempotency: Idempotency,
create_client: impl Fn(tonic::transport::Channel) -> GrpcClient,
call: impl AsyncFn(&mut GrpcClient, tonic::Request<Request>) -> Result<tonic::Response<R>, tonic::Status>,
request: Request,
) -> Result<R, tonic::Status>
where
Request: Clone,
GrpcClient: ServiceClient,
{
const GRPC_TIMEOUT_LIMIT: std::time::Duration = std::time::Duration::from_secs(100_000_000 * 60 * 60);
let deadline: Option<std::time::Instant> = match &self.retry_policy {
RetryPolicy::WithDeadline(d) if *d < GRPC_TIMEOUT_LIMIT => std::time::Instant::now().checked_add(*d),
RetryPolicy::WithDeadline(_) | RetryPolicy::Never | RetryPolicy::Forever => None,
};
let timing_allows_retry = || match &self.retry_policy {
RetryPolicy::Never => false,
RetryPolicy::Forever => true,
RetryPolicy::WithDeadline(_) => deadline.is_none_or(|d| std::time::Instant::now() <= d),
};
let remaining = || deadline.map(|d| d.saturating_duration_since(std::time::Instant::now()));
let mut client = self.service_client(create_client);
let mut token_refreshed = false;
loop {
let metric = metrics::MetricsSpan::new(&self.metrics);
let mut call_request = tonic::Request::new(request.clone());
if let Some(timeout) = remaining() {
call_request.set_timeout(timeout);
}
let token_generation = self.attach_token(&mut call_request);
let response = call(&mut client, call_request).await;
metric.complete(response.is_ok());
match response {
Ok(r) => break Ok(r.into_inner()),
Err(e) => {
if !token_refreshed
&& self.auth.is_some()
&& Self::is_stale_token_error(&e)
&& self.refresh_auth_token(token_generation, deadline).await.is_ok()
{
token_refreshed = true;
continue;
}
if Self::can_retry(&e, idempotency) && timing_allows_retry() {
continue;
}
return Err(e);
}
}
}
}
fn attach_token<T>(&self, request: &mut tonic::Request<T>) -> u64 {
let Some(auth) = &self.auth else {
return 0;
};
let state = auth.token.read().unwrap();
if let Some(token) = &state.1 {
request.metadata_mut().insert("token", token.clone());
}
state.0
}
async fn attach_stream_token<T>(&self, request: &mut tonic::Request<T>, deadline: std::time::Instant) -> u64 {
if let Some(auth) = &self.auth
&& auth.credentials.is_some()
{
let (generation, cached) = {
let state = auth.token.read().unwrap();
(state.0, state.1.is_some())
};
if !cached {
let _ = self.refresh_auth_token_unserialized(generation, Some(deadline)).await;
}
}
self.attach_token(request)
}
fn can_retry(status: &tonic::Status, idempotency: Idempotency) -> bool {
match idempotency {
Idempotency::Immutable => {
status.code() == tonic::Code::Unavailable
|| is_client_side_timeout(status)
|| is_transport_error(status)
}
Idempotency::Mutable => is_connect_error(status),
}
}
fn is_stale_token_error(status: &tonic::Status) -> bool {
match status.code() {
tonic::Code::Unauthenticated => true,
tonic::Code::InvalidArgument => matches!(
status.message(),
"etcdserver: user name is empty" | "etcdserver: revision of auth store is old"
),
_ => false,
}
}
fn is_stale_token_refusal(reason: &str) -> bool {
matches!(
reason,
"rpc error: code = Unauthenticated desc = etcdserver: invalid auth token"
| "rpc error: code = InvalidArgument desc = etcdserver: user name is empty"
| "rpc error: code = InvalidArgument desc = etcdserver: revision of auth store is old"
)
}
}
#[cfg(test)]
mod test {
use super::{Idempotency::*, *};
fn connect_failure() -> tonic::Status {
tonic::Status::from_error(Box::new(tonic::ConnectError(Box::new(std::io::Error::from(
std::io::ErrorKind::ConnectionRefused,
)))))
}
fn dropped_connection() -> tonic::Status {
#[derive(Debug)]
struct TransportError;
impl std::fmt::Display for TransportError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("transport error")
}
}
impl std::error::Error for TransportError {}
tonic::Status::from_error(Box::new(TransportError))
}
#[test]
fn connect_failure_is_recognized() {
let status = connect_failure();
assert_eq!(status.code(), tonic::Code::Unavailable, "{status:?}");
assert!(is_connect_error(&status), "{status:?}");
}
#[test]
fn never_sent_retries_for_both_classes() {
let status = connect_failure();
assert!(ClientInner::can_retry(&status, Immutable));
assert!(ClientInner::can_retry(&status, Mutable));
}
#[test]
fn dropped_connection_retries_only_reads() {
let status = dropped_connection();
assert!(is_transport_error(&status), "{status:?}");
assert!(!is_connect_error(&status), "{status:?}");
assert!(ClientInner::can_retry(&status, Immutable));
assert!(
!ClientInner::can_retry(&status, Mutable),
"a mutation may already have applied"
);
}
#[test]
fn server_sent_unavailable_retries_only_reads() {
let status = tonic::Status::new(tonic::Code::Unavailable, "etcdserver: no leader");
assert!(!is_connect_error(&status), "{status:?}");
assert!(ClientInner::can_retry(&status, Immutable));
assert!(!ClientInner::can_retry(&status, Mutable));
}
#[test]
fn client_side_timeout_retries_only_reads() {
let status = tonic::Status::new(tonic::Code::Cancelled, "Timeout expired");
assert!(is_client_side_timeout(&status));
assert!(ClientInner::can_retry(&status, Immutable));
assert!(!ClientInner::can_retry(&status, Mutable));
}
#[test]
fn non_transient_errors_never_retry() {
for status in [
tonic::Status::new(tonic::Code::NotFound, "etcdserver: member not found"),
tonic::Status::new(tonic::Code::FailedPrecondition, "etcdserver: ID exists"),
tonic::Status::new(tonic::Code::InvalidArgument, "etcdserver: key is not provided"),
] {
assert!(!ClientInner::can_retry(&status, Immutable), "{status:?}");
assert!(!ClientInner::can_retry(&status, Mutable), "{status:?}");
}
}
#[test]
fn missing_token_is_a_stale_token() {
let status = tonic::Status::invalid_argument("etcdserver: user name is empty");
assert!(ClientInner::is_stale_token_error(&status));
}
#[test]
fn rejected_token_is_a_stale_token() {
let status = tonic::Status::unauthenticated("etcdserver: invalid auth token");
assert!(ClientInner::is_stale_token_error(&status));
}
#[test]
fn superseded_token_is_a_stale_token() {
let status = tonic::Status::invalid_argument("etcdserver: revision of auth store is old");
assert!(ClientInner::is_stale_token_error(&status));
}
#[test]
fn permission_denied_is_not_a_stale_token() {
let status = tonic::Status::permission_denied("etcdserver: permission denied");
assert!(!ClientInner::is_stale_token_error(&status));
assert!(!ClientInner::can_retry(&status, Immutable));
assert!(!ClientInner::can_retry(&status, Mutable));
}
#[test]
fn unrelated_invalid_arguments_are_not_stale_tokens() {
for status in [
tonic::Status::invalid_argument("etcdserver: invalid auth management"),
tonic::Status::invalid_argument("etcdserver: key is not provided"),
] {
assert!(!ClientInner::is_stale_token_error(&status), "{status:?}");
}
}
#[test]
fn stale_token_refusals_are_recognized() {
for reason in [
"rpc error: code = Unauthenticated desc = etcdserver: invalid auth token",
"rpc error: code = InvalidArgument desc = etcdserver: user name is empty",
"rpc error: code = InvalidArgument desc = etcdserver: revision of auth store is old",
] {
assert!(ClientInner::is_stale_token_refusal(reason), "{reason}");
}
}
#[test]
fn other_refusals_are_not_stale_tokens() {
for reason in [
"rpc error: code = PermissionDenied desc = etcdserver: permission denied",
"mvcc: watcher range is empty",
"etcdserver: mvcc: required revision has been compacted",
"rpc error: code = InvalidArgument desc = etcdserver: key is not provided",
"rpc error: code = Unauthenticated",
"",
] {
assert!(!ClientInner::is_stale_token_refusal(reason), "{reason}");
}
}
#[tokio::test]
async fn streams_authenticate_without_the_refresh_lock() {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let client = Client::builder()
.add_connection(format!("http://{}", listener.local_addr().unwrap()))
.unwrap()
.credentials("root", "rootpw")
.build()
.unwrap();
let mut request = tonic::Request::new(());
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
let mut attach = std::pin::pin!(client.inner.attach_stream_token(&mut request, deadline));
let polled = std::future::poll_fn(|cx| std::task::Poll::Ready(attach.as_mut().poll(cx))).await;
assert!(polled.is_pending());
assert!(client.inner.auth.as_ref().unwrap().refresh.try_lock().is_ok());
}
#[test]
fn server_cannot_forge_a_connect_failure() {
let status = tonic::Status::new(tonic::Code::Unavailable, "tcp connect error");
assert!(!is_connect_error(&status), "{status:?}");
assert!(!ClientInner::can_retry(&status, Mutable));
}
}