#[cfg(test)]
use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
use crate::cluster::rpc::http_auth::{
AuthenticatedPeerReplayCapabilities, RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER,
RPC_CONTENT_SHA256_HEADER, RPC_REPLAY_CACHE_CAPABILITY_HEADER, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
RollingMutationBodyDigest, TIMESTAMP_HEADER, internode_rpc_body_digest_strict,
verify_tonic_peer_replay_capabilities_response,
};
use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience};
#[cfg(test)]
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
use crate::runtime::sources as runtime_sources;
use http::{Request as HttpRequest, Response as HttpResponse, Uri};
use rustfs_protos::{
ChannelClass, create_new_channel, get_channel_for_class,
proto_gen::node_service::{
heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient,
scanner_control_service_client::ScannerControlServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
},
};
use std::{
collections::HashMap,
error::Error,
future::Future,
io::ErrorKind,
pin::Pin,
sync::{LazyLock, Mutex},
task::{Context, Poll},
};
use tonic::{service::interceptor::InterceptedService, transport::Channel};
use tower::Service;
use tracing::debug;
use uuid::Uuid;
use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
pub async fn node_service_time_out_client(
addr: &String,
interceptor: TonicInterceptor,
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
}
pub(crate) async fn scanner_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> crate::error::Result<ScannerControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr)
.await
.map_err(|err| crate::error::Error::other(err.to_string()))?,
};
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(limit)
.max_encoding_message_size(limit))
}
pub async fn heal_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> Result<HealControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr).await?,
};
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
}
pub async fn tier_mutation_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr).await?,
};
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
}
pub async fn node_service_time_out_client_for_class(
addr: &String,
interceptor: TonicInterceptor,
class: ChannelClass,
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match class {
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
Some(channel) => {
debug!("Using cached gRPC channel for: {}", addr);
channel
}
None => create_new_channel(addr).await?,
},
ChannelClass::Bulk => get_channel_for_class(addr, ChannelClass::Bulk).await?,
};
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
}
pub async fn node_service_time_out_client_no_auth(
addr: &String,
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
}
pub(crate) fn embedded_tonic_status(io_err: &std::io::Error) -> Option<&tonic::Status> {
io_err.get_ref()?.downcast_ref::<RpcStatusError>().map(RpcStatusError::status)
}
pub(crate) fn is_network_like_status(status: &tonic::Status) -> bool {
match status.code() {
tonic::Code::Unavailable => true,
tonic::Code::Unknown => message_has_network_needle(&status.to_string()),
_ => false,
}
}
pub(crate) fn message_has_network_needle(message: &str) -> bool {
let message = message.to_ascii_lowercase();
[
"temporarily offline",
"transport error",
"code: 'the service is currently unavailable'",
"status: unavailable",
"error trying to connect",
"connection refused",
"connection reset",
"broken pipe",
"not connected",
"unexpected eof",
"timed out",
"deadline has elapsed",
"connection closed",
"connection aborted",
"tcp connect error",
]
.iter()
.any(|needle| message.contains(needle))
}
pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
match err {
DiskError::Timeout => true,
DiskError::RemoteClientUnavailable(detail) => message_has_network_needle(detail),
DiskError::Io(io_err) => {
if let Some(status) = embedded_tonic_status(io_err) {
return is_network_like_status(status);
}
if matches!(
io_err.kind(),
ErrorKind::TimedOut
| ErrorKind::ConnectionRefused
| ErrorKind::ConnectionReset
| ErrorKind::BrokenPipe
| ErrorKind::NotConnected
| ErrorKind::ConnectionAborted
| ErrorKind::UnexpectedEof
) {
return true;
}
message_has_network_needle(&io_err.to_string())
}
_ => false,
}
}
#[derive(Clone, Debug)]
pub struct ReplayScopeChannel<S> {
inner: S,
audience: Option<String>,
}
pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum PeerReplayCapability {
Capable { boot_epoch: Uuid },
Revoked,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
struct PeerReplayState {
boot_epoch: Option<Uuid>,
cache_capability: Option<PeerReplayCapability>,
}
#[derive(Clone, Copy, Debug)]
struct PeerReplayStateSnapshot(PeerReplayState);
static PEER_REPLAY_STATES: LazyLock<Mutex<HashMap<String, PeerReplayState>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
impl<S> ReplayScopeChannel<S> {
fn new(inner: S, audience: Option<String>) -> Self {
Self { inner, audience }
}
}
#[allow(dead_code, reason = "replay-state probe asserted by this file's tests (backlog#1823)")]
fn peer_replay_state(audience: &str) -> PeerReplayState {
PEER_REPLAY_STATES
.lock()
.ok()
.and_then(|states| states.get(audience).copied())
.unwrap_or_default()
}
pub(crate) fn clear_peer_replay_state_for_addr(addr: &str) -> std::io::Result<()> {
let uri = addr
.parse::<Uri>()
.map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?;
let audience = uri
.authority()
.map(|authority| normalize_tonic_rpc_audience(authority.as_str()))
.ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))??;
if let Ok(mut states) = PEER_REPLAY_STATES.lock() {
states.remove(&audience);
}
Ok(())
}
fn apply_peer_replay_response(
audience: String,
sent_state: PeerReplayState,
response: std::io::Result<AuthenticatedPeerReplayCapabilities>,
) {
if let Ok(mut states) = PEER_REPLAY_STATES.lock() {
let current_state = states.get(&audience).copied().unwrap_or_default();
let mut next_state = current_state;
if let Ok(response) = &response
&& sent_state.boot_epoch == current_state.boot_epoch
{
next_state.boot_epoch = Some(response.boot_epoch);
}
if sent_state.boot_epoch == current_state.boot_epoch {
let response_capability = response
.as_ref()
.ok()
.filter(|response| response.dynamic_replay_cache)
.map(|response| response.boot_epoch);
match (sent_state.cache_capability, current_state.cache_capability, response_capability) {
(None, None, Some(boot_epoch))
| (Some(PeerReplayCapability::Revoked), Some(PeerReplayCapability::Revoked), Some(boot_epoch)) => {
next_state.cache_capability = Some(PeerReplayCapability::Capable { boot_epoch });
}
(
Some(PeerReplayCapability::Capable {
boot_epoch: sent_boot_epoch,
}),
Some(PeerReplayCapability::Capable {
boot_epoch: current_boot_epoch,
}),
Some(response_boot_epoch),
) if sent_boot_epoch == current_boot_epoch => {
next_state.cache_capability = Some(PeerReplayCapability::Capable {
boot_epoch: response_boot_epoch,
});
}
(
Some(PeerReplayCapability::Capable {
boot_epoch: sent_boot_epoch,
}),
Some(PeerReplayCapability::Capable {
boot_epoch: current_boot_epoch,
}),
None,
) if sent_boot_epoch == current_boot_epoch => {
next_state.cache_capability = Some(PeerReplayCapability::Revoked);
}
_ => {}
}
}
states.insert(audience, next_state);
}
}
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
where
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
S::Error: Send + 'static,
S::Future: Send + 'static,
ReqBody: Send + 'static,
ResBody: Send + 'static,
{
type Response = HttpResponse<ResBody>;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
let authenticated = self.audience.as_ref().is_some_and(|_| {
request
.headers()
.get(RPC_AUTH_VERSION_HEADER)
.and_then(|value| value.to_str().ok())
== Some(RPC_AUTH_VERSION_V2)
});
let challenge = authenticated.then(Uuid::new_v4);
let sent_state = request
.extensions()
.get::<PeerReplayStateSnapshot>()
.map(|snapshot| snapshot.0)
.unwrap_or_default();
if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
request.headers_mut().insert(
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
challenge.to_string().parse().expect("UUID must be a valid header value"),
);
if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
sent_state.boot_epoch,
request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
request
.headers()
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok()),
) {
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
Ok(headers) => request.headers_mut().extend(headers),
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
}
}
}
let audience = self.audience.clone();
let future = self.inner.call(request);
Box::pin(async move {
let response = future.await?;
if let (Some(audience), Some(challenge)) = (audience, challenge) {
let response_state = verify_tonic_peer_replay_capabilities_response(&audience, challenge, response.headers());
if let Err(error) = &response_state
&& (response.headers().contains_key(RPC_REPLAY_CACHE_CAPABILITY_HEADER)
|| response.headers().contains_key(RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER))
{
debug!(
event = "internode_rpc_capability_proof_rejected",
component = "ecstore",
subsystem = "rpc_client",
result = "rejected",
error = %error,
"internode RPC capability proof rejected"
)
}
apply_peer_replay_response(audience, sent_state, response_state);
}
Ok(response)
})
}
}
pub struct TonicSignatureInterceptor {
audience: Option<String>,
body_digest_strict: bool,
}
impl tonic::service::Interceptor for TonicSignatureInterceptor {
fn call(&mut self, mut req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
let method = req
.extensions()
.get::<tonic::GrpcMethod<'_>>()
.ok_or_else(|| tonic::Status::unauthenticated("Missing gRPC method metadata"))?;
let audience = self
.audience
.as_deref()
.ok_or_else(|| tonic::Status::unauthenticated("Missing gRPC audience"))?;
let content_sha256 = req
.metadata()
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok());
let rolling_mutation = req.extensions().get::<RollingMutationBodyDigest>().is_some();
let peer_state = PEER_REPLAY_STATES
.lock()
.map_err(|_| tonic::Status::unauthenticated("RPC peer capability state unavailable"))?
.get(audience)
.copied()
.unwrap_or_default();
let content_sha256 = if content_sha256.is_some() {
if peer_state.cache_capability == Some(PeerReplayCapability::Revoked) {
return Err(tonic::Status::unauthenticated("RPC peer replay capability changed"));
}
if rolling_mutation && !self.body_digest_strict && peer_state.boot_epoch.is_none() {
None
} else {
content_sha256
}
} else {
content_sha256
};
let headers = gen_tonic_signature_headers(audience, method.service(), method.method(), content_sha256)
.map_err(|_| tonic::Status::unauthenticated("No valid auth token"))?;
req.metadata_mut().as_mut().extend(headers);
req.extensions_mut().insert(PeerReplayStateSnapshot(peer_state));
inject_trace_context_into_metadata(req.metadata_mut());
inject_request_id_into_metadata(req.metadata_mut());
Ok(req)
}
}
pub fn gen_tonic_signature_interceptor() -> TonicSignatureInterceptor {
TonicSignatureInterceptor {
audience: None,
body_digest_strict: internode_rpc_body_digest_strict(),
}
}
pub struct NoOpInterceptor;
impl tonic::service::Interceptor for NoOpInterceptor {
fn call(&mut self, req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
Ok(req)
}
}
pub enum TonicInterceptor {
Signature(TonicSignatureInterceptor),
NoOp(NoOpInterceptor),
}
impl TonicInterceptor {
fn with_rpc_audience(mut self, addr: &str) -> std::io::Result<Self> {
if let Self::Signature(interceptor) = &mut self {
let uri = addr
.parse::<Uri>()
.map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?;
let audience = uri
.authority()
.map(|authority| normalize_tonic_rpc_audience(authority.as_str()))
.ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))?;
interceptor.audience = Some(audience?);
}
Ok(self)
}
fn replay_scope_audience(&self) -> Option<String> {
match self {
Self::Signature(interceptor) => interceptor.audience.clone(),
Self::NoOp(_) => None,
}
}
}
impl tonic::service::Interceptor for TonicInterceptor {
fn call(&mut self, req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
match self {
TonicInterceptor::Signature(interceptor) => interceptor.call(req),
TonicInterceptor::NoOp(interceptor) => interceptor.call(req),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use opentelemetry::global;
use opentelemetry::trace::{SpanContext, TraceContextExt, TraceFlags, TraceId, TraceState, TracerProvider as _};
use opentelemetry_sdk::propagation::TraceContextPropagator;
use opentelemetry_sdk::trace::SdkTracerProvider;
use tonic::service::Interceptor;
use tracing_opentelemetry::OpenTelemetrySpanExt;
use tracing_subscriber::{Registry, layer::SubscriberExt};
#[derive(Clone)]
struct EpochProofService {
audience: String,
include_capability: bool,
seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
}
impl Service<HttpRequest<()>> for EpochProofService {
type Response = HttpResponse<()>;
type Error = std::convert::Infallible;
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, request: HttpRequest<()>) -> Self::Future {
self.seen_headers
.lock()
.expect("test header capture lock must not be poisoned")
.push(request.headers().clone());
let challenge = tonic_boot_epoch_challenge(request.headers())
.expect("client challenge must be syntactically valid")
.expect("authenticated client request must carry a boot epoch challenge");
let mut response = HttpResponse::new(());
let mut headers = tonic_boot_epoch_response_headers(&self.audience, challenge)
.expect("test server must be able to sign an epoch proof");
if !self.include_capability {
headers.remove(RPC_REPLAY_CACHE_CAPABILITY_HEADER);
headers.remove(RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER);
}
response.headers_mut().extend(headers);
std::future::ready(Ok(response))
}
}
#[derive(Clone)]
struct MissingProofService;
impl Service<HttpRequest<()>> for MissingProofService {
type Response = HttpResponse<()>;
type Error = std::convert::Infallible;
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, _request: HttpRequest<()>) -> Self::Future {
std::future::ready(Ok(HttpResponse::new(())))
}
}
fn ensure_test_rpc_secret() {
runtime_sources::ensure_test_rpc_secret();
}
fn test_request() -> tonic::Request<()> {
test_request_for("Ping")
}
fn test_request_for(method: &'static str) -> tonic::Request<()> {
let mut request = tonic::Request::new(());
request
.extensions_mut()
.insert(tonic::GrpcMethod::new("node_service.NodeService", method));
request
}
fn test_interceptor() -> TonicSignatureInterceptor {
test_interceptor_for("node-a:9000", false)
}
fn test_interceptor_for(audience: &str, body_digest_strict: bool) -> TonicSignatureInterceptor {
TonicSignatureInterceptor {
audience: Some(audience.to_string()),
body_digest_strict,
}
}
fn clear_peer_capability(audience: &str) {
PEER_REPLAY_STATES
.lock()
.expect("peer capability cache lock must not be poisoned")
.remove(audience);
}
fn set_peer_capability(audience: &str, state: PeerReplayState) {
PEER_REPLAY_STATES
.lock()
.expect("peer capability cache lock must not be poisoned")
.insert(audience.to_string(), state);
}
fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> {
let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest {
args: "canonical mutation request".to_string(),
});
request
.extensions_mut()
.insert(tonic::GrpcMethod::new("node_service.NodeService", method));
crate::cluster::rpc::set_tonic_rolling_mutation_body_digest(&mut request).expect("test mutation digest must be attached");
request.map(|_| ())
}
fn replay_scope_request(audience: &str, method: &'static str) -> HttpRequest<()> {
let mut request = HttpRequest::builder()
.uri(format!("/node_service.NodeService/{method}"))
.body(())
.expect("test RPC request must build");
request.headers_mut().extend(
gen_tonic_signature_headers(audience, "node_service.NodeService", method, None).expect("v2 test headers must mint"),
);
request
.extensions_mut()
.insert(PeerReplayStateSnapshot(peer_replay_state(audience)));
request
}
fn authenticated_peer_response(boot_epoch: Uuid, dynamic_replay_cache: bool) -> AuthenticatedPeerReplayCapabilities {
AuthenticatedPeerReplayCapabilities {
boot_epoch,
dynamic_replay_cache,
}
}
fn with_trace_parent<F>(trace_id_hex: &str, f: F)
where
F: FnOnce(),
{
global::set_text_map_propagator(TraceContextPropagator::new());
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("rpc-client-tests");
let subscriber = Registry::default().with(tracing_opentelemetry::layer().with_tracer(tracer));
tracing::subscriber::with_default(subscriber, || {
let span = tracing::info_span!("rpc-client-test-span");
let trace_id = TraceId::from_hex(trace_id_hex).expect("trace id should be valid hex");
let span_id = opentelemetry::trace::SpanId::from_hex("0102030405060708").expect("span id should be valid hex");
let parent = SpanContext::new(trace_id, span_id, TraceFlags::SAMPLED, true, TraceState::default());
span.set_parent(opentelemetry::Context::new().with_remote_span_context(parent))
.expect("failed to set parent context");
let _guard = span.enter();
f();
});
let _ = provider.shutdown();
}
#[test]
fn network_like_disk_error_uses_typed_status_code() {
assert!(is_network_like_disk_error(&DiskError::from(tonic::Status::unavailable(
"storage layer is not initialized"
))));
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::internal(
"failed to heal bucket \"unavailable-logs\""
))));
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::unauthenticated(
"No valid auth token"
))));
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::cancelled("Timeout expired"))));
}
#[test]
fn embedded_tonic_status_is_recovered_across_error_conversions() {
let from_storage: DiskErrorType = crate::error::Error::from(tonic::Status::unavailable("peer gone"))
.narrow_to_disk()
.expect("status-derived Io errors narrow through the bridge");
let DiskError::Io(io_err) = &from_storage else {
panic!("status-derived disk error should stay an Io error");
};
assert_eq!(embedded_tonic_status(io_err).map(|status| status.code()), Some(tonic::Code::Unavailable));
let from_disk = crate::error::Error::from(DiskError::from(tonic::Status::unavailable("peer gone")));
let crate::error::Error::Io(io_err) = &from_disk else {
panic!("status-derived storage error should stay an Io error");
};
assert_eq!(embedded_tonic_status(io_err).map(|status| status.code()), Some(tonic::Code::Unavailable));
}
#[test]
fn network_like_disk_error_ignores_transport_words_in_application_statuses() {
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::internal(
"connection refused while dialing downstream backend"
))));
assert!(!is_network_like_disk_error(&DiskError::from(tonic::Status::unauthenticated(
"connection reset while validating token"
))));
}
#[test]
fn network_like_disk_error_requires_anchored_unavailable_needle() {
assert!(!is_network_like_disk_error(&DiskError::other("bucket \"unavailable-logs\" not found")));
assert!(is_network_like_disk_error(&DiskError::other(
"code: 'The service is currently unavailable', message: \"peer gone\""
)));
assert!(is_network_like_disk_error(&DiskError::other(
"status: Unavailable, message: \"peer gone\""
)));
assert!(is_network_like_disk_error(&DiskError::other("connection refused")));
assert!(!is_network_like_disk_error(&DiskError::FileNotFound));
}
#[test]
fn network_like_disk_error_keeps_typed_client_failures_classified() {
assert!(is_network_like_disk_error(&DiskError::RemoteClientUnavailable(
"transport error: connection refused".to_string()
)));
assert!(!is_network_like_disk_error(&DiskError::RemoteClientUnavailable(
"invalid client credentials".to_string()
)));
}
#[test]
fn test_signature_interceptor_keeps_auth_headers() {
ensure_test_rpc_secret();
let mut interceptor = test_interceptor();
let req = test_request();
let req = interceptor.call(req).expect("interceptor call should succeed");
assert!(req.metadata().contains_key("x-rustfs-signature"));
assert!(req.metadata().contains_key("x-rustfs-timestamp"));
assert!(req.metadata().contains_key("x-rustfs-rpc-signature-v2"));
assert!(req.metadata().contains_key("x-rustfs-rpc-nonce"));
assert!(
crate::cluster::rpc::verify_tonic_rpc_signature(
"node-a:9000",
"/node_service.NodeService/Ping",
req.metadata().as_ref(),
)
.is_ok(),
"interceptor signature should bind the configured peer audience and generated method"
);
}
#[test]
fn unknown_peer_mutations_use_cache_free_unsigned_v2() {
ensure_test_rpc_secret();
let audience = "legacy-body-digest-client-test:9000";
clear_peer_capability(audience);
let mut interceptor = test_interceptor_for(audience, false);
for method in ["Lock", "WriteAll"] {
let request = interceptor
.call(rolling_mutation_request(method))
.expect("interceptor call should succeed");
assert_eq!(
request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok()),
Some("UNSIGNED-PAYLOAD")
);
assert_eq!(
request
.metadata()
.get("x-rustfs-rpc-nonce")
.and_then(|value| value.to_str().ok()),
Some("unsigned")
);
assert!(
crate::cluster::rpc::verify_tonic_rpc_signature(
audience,
&format!("/node_service.NodeService/{method}"),
request.metadata().as_ref(),
)
.is_ok(),
"the cache-free request must retain valid audience- and method-bound v2 authentication"
);
}
}
#[test]
fn unknown_peer_exact_body_contract_remains_body_bound() {
ensure_test_rpc_secret();
let audience = "exact-body-contract-client-test:9000";
clear_peer_capability(audience);
let mut interceptor = test_interceptor_for(audience, false);
let mut request = test_request_for("ScannerActivity");
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, b"exact scanner activity body")
.expect("test exact body digest must be attached");
let expected_digest = request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok())
.expect("test request must carry its digest")
.to_string();
let request = interceptor.call(request).expect("interceptor call should succeed");
assert_eq!(
request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok()),
Some(expected_digest.as_str())
);
assert!(
crate::cluster::rpc::verify_tonic_rpc_signature(
audience,
"/node_service.NodeService/ScannerActivity",
request.metadata().as_ref(),
)
.is_ok()
);
}
#[test]
fn unknown_peer_iam_mutation_helper_remains_body_bound() {
ensure_test_rpc_secret();
let audience = "exact-iam-mutation-client-test:9000";
clear_peer_capability(audience);
let mut interceptor = test_interceptor_for(audience, false);
let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::DeleteUserRequest {
access_key: "target-access-key".to_string(),
});
request
.extensions_mut()
.insert(tonic::GrpcMethod::new("node_service.NodeService", "DeleteUser"));
crate::cluster::rpc::set_tonic_mutation_body_digest(&mut request).expect("test IAM mutation digest must be attached");
let request = request.map(|_| ());
let expected_digest = request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok())
.expect("test IAM mutation must carry its digest")
.to_string();
let request = interceptor.call(request).expect("interceptor call should succeed");
assert_eq!(
request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok()),
Some(expected_digest.as_str())
);
assert!(
crate::cluster::rpc::verify_tonic_rpc_signature(
audience,
"/node_service.NodeService/DeleteUser",
request.metadata().as_ref(),
)
.is_ok(),
"IAM mutations must remain body-bound before capability discovery"
);
}
#[test]
fn authenticated_replay_cache_capability_enables_body_binding() {
ensure_test_rpc_secret();
let audience = "body-digest-capable-client-test:9000";
clear_peer_capability(audience);
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
let service = EpochProofService {
audience: audience.to_string(),
include_capability: true,
seen_headers,
};
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
.expect("authenticated capability probe must complete");
let mut interceptor = test_interceptor_for(audience, false);
let request = rolling_mutation_request("Lock");
let expected_digest = request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok())
.expect("test mutation must carry its digest")
.to_string();
let request = interceptor.call(request).expect("interceptor call should succeed");
assert_eq!(
request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok()),
Some(expected_digest.as_str())
);
let nonce = request
.metadata()
.get("x-rustfs-rpc-nonce")
.and_then(|value| value.to_str().ok())
.and_then(|value| Uuid::parse_str(value).ok())
.expect("capable peer body-bound mutation must carry a UUID nonce");
assert!(!nonce.is_nil());
assert!(
crate::cluster::rpc::verify_tonic_rpc_signature(
audience,
"/node_service.NodeService/Lock",
request.metadata().as_ref(),
)
.is_ok(),
"the body-bound request must retain valid audience- and method-bound v2 authentication"
);
clear_peer_capability(audience);
}
#[test]
fn invalid_capability_proof_does_not_enable_body_binding() {
ensure_test_rpc_secret();
let audience = "invalid-capability-client-test:9000";
clear_peer_capability(audience);
let service = EpochProofService {
audience: "wrong-capability-audience:9000".to_string(),
include_capability: true,
seen_headers: std::sync::Arc::new(Mutex::new(Vec::new())),
};
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
.expect("invalid capability response must still complete");
let mut interceptor = test_interceptor_for(audience, false);
let request = interceptor
.call(rolling_mutation_request("Lock"))
.expect("legacy-compatible mutation must still be signed");
assert_eq!(
request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok()),
Some("UNSIGNED-PAYLOAD")
);
}
#[test]
fn legacy_boot_proof_keeps_mutations_body_bound_and_enables_non_ping_v3() {
ensure_test_rpc_secret();
let audience = "legacy-boot-proof-client-test:9000";
clear_peer_capability(audience);
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
let service = EpochProofService {
audience: audience.to_string(),
include_capability: false,
seen_headers: seen_headers.clone(),
};
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
.expect("legacy boot proof response must complete");
let state = peer_replay_state(audience);
assert!(state.boot_epoch.is_some(), "authenticated legacy proof must enable replay-scoped v3");
assert_eq!(state.cache_capability, None);
let mut interceptor = test_interceptor_for(audience, false);
let request = rolling_mutation_request("Lock");
let expected_digest = request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok())
.expect("test mutation must carry its digest")
.to_string();
let request = interceptor.call(request).expect("legacy-compatible mutation must be signed");
assert_eq!(
request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok()),
Some(expected_digest.as_str())
);
let (metadata, extensions, body) = request.into_parts();
let mut request = HttpRequest::new(body);
*request.uri_mut() = "/node_service.NodeService/Lock".parse().expect("test RPC URI must parse");
*request.headers_mut() = metadata.into_headers();
*request.extensions_mut() = extensions;
futures::executor::block_on(channel.call(request)).expect("legacy strict-compatible lock request must complete");
let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
assert!(
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
"authenticated legacy boot proof must enable v3 on a non-Ping request"
);
}
#[test]
fn reordered_capability_responses_cannot_undo_newer_state() {
let audience = "reordered-capability-client-test:9000";
let epoch_one = Uuid::new_v4();
let epoch_two = Uuid::new_v4();
clear_peer_capability(audience);
let unknown = PeerReplayState::default();
apply_peer_replay_response(audience.to_string(), unknown, Ok(authenticated_peer_response(epoch_one, true)));
apply_peer_replay_response(audience.to_string(), unknown, Err(std::io::Error::other("delayed legacy response")));
let epoch_one_state = PeerReplayState {
boot_epoch: Some(epoch_one),
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch: epoch_one }),
};
assert_eq!(peer_replay_state(audience), epoch_one_state);
apply_peer_replay_response(audience.to_string(), epoch_one_state, Err(std::io::Error::other("rollback response")));
apply_peer_replay_response(audience.to_string(), epoch_one_state, Ok(authenticated_peer_response(epoch_one, true)));
assert_eq!(
peer_replay_state(audience),
PeerReplayState {
boot_epoch: Some(epoch_one),
cache_capability: Some(PeerReplayCapability::Revoked),
}
);
let revoked = peer_replay_state(audience);
apply_peer_replay_response(audience.to_string(), revoked, Ok(authenticated_peer_response(epoch_two, true)));
apply_peer_replay_response(audience.to_string(), epoch_one_state, Ok(authenticated_peer_response(epoch_one, true)));
assert_eq!(
peer_replay_state(audience),
PeerReplayState {
boot_epoch: Some(epoch_two),
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch: epoch_two }),
}
);
clear_peer_capability(audience);
}
#[test]
fn stale_capability_response_cannot_cross_a_new_boot_epoch() {
let audience = "cross-epoch-capability-client-test:9000";
let epoch_one = Uuid::new_v4();
let epoch_two = Uuid::new_v4();
let epoch_three = Uuid::new_v4();
clear_peer_capability(audience);
let revoked_epoch_one = PeerReplayState {
boot_epoch: Some(epoch_one),
cache_capability: Some(PeerReplayCapability::Revoked),
};
PEER_REPLAY_STATES
.lock()
.expect("peer replay state lock must not be poisoned")
.insert(audience.to_string(), revoked_epoch_one);
apply_peer_replay_response(
audience.to_string(),
revoked_epoch_one,
Ok(authenticated_peer_response(epoch_three, false)),
);
apply_peer_replay_response(audience.to_string(), revoked_epoch_one, Ok(authenticated_peer_response(epoch_two, true)));
assert_eq!(
peer_replay_state(audience),
PeerReplayState {
boot_epoch: Some(epoch_three),
cache_capability: Some(PeerReplayCapability::Revoked),
},
"a stale dynamic-cache proof must not cross a newer authenticated boot epoch"
);
clear_peer_capability(audience);
}
#[test]
fn clear_peer_replay_state_for_addr_removes_normalized_audience() {
let audience = "clear-peer-replay-state-test:9000";
let boot_epoch = Uuid::new_v4();
set_peer_capability(
audience,
PeerReplayState {
boot_epoch: Some(boot_epoch),
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
},
);
clear_peer_replay_state_for_addr("http://clear-peer-replay-state-test:9000").expect("peer URI should clear replay state");
assert_eq!(peer_replay_state(audience), PeerReplayState::default());
}
#[test]
fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() {
ensure_test_rpc_secret();
let audience = "capability-snapshot-client-test:9000";
clear_peer_capability(audience);
let boot_epoch = Uuid::new_v4();
let mut interceptor = test_interceptor_for(audience, false);
let request = interceptor
.call(rolling_mutation_request("Lock"))
.expect("legacy-compatible request must pass the interceptor");
assert_eq!(
request
.extensions()
.get::<PeerReplayStateSnapshot>()
.map(|snapshot| snapshot.0),
Some(PeerReplayState::default()),
"interceptor must preserve its unknown-state admission snapshot"
);
let capable_state = PeerReplayState {
boot_epoch: Some(boot_epoch),
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
};
PEER_REPLAY_STATES
.lock()
.expect("peer capability cache lock must not be poisoned")
.insert(audience.to_string(), capable_state);
let (metadata, extensions, body) = request.into_parts();
let mut request = HttpRequest::new(body);
*request.uri_mut() = "/node_service.NodeService/Lock".parse().expect("test RPC URI must parse");
*request.headers_mut() = metadata.into_headers();
*request.extensions_mut() = extensions;
let mut channel = ReplayScopeChannel::new(MissingProofService, Some(audience.to_string()));
futures::executor::block_on(channel.call(request)).expect("in-flight request response must complete");
assert_eq!(peer_replay_state(audience), capable_state);
clear_peer_capability(audience);
}
#[test]
fn strict_mode_keeps_unknown_peer_mutations_body_bound() {
ensure_test_rpc_secret();
let audience = "strict-body-digest-client-test:9000";
clear_peer_capability(audience);
let mut interceptor = test_interceptor_for(audience, true);
let request = rolling_mutation_request("WriteAll");
let expected_digest = request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok())
.expect("test mutation must carry its digest")
.to_string();
let request = interceptor.call(request).expect("interceptor call should succeed");
assert_eq!(
request
.metadata()
.get("x-rustfs-content-sha256")
.and_then(|value| value.to_str().ok()),
Some(expected_digest.as_str())
);
let nonce = request
.metadata()
.get("x-rustfs-rpc-nonce")
.and_then(|value| value.to_str().ok())
.and_then(|value| Uuid::parse_str(value).ok())
.expect("strict body-bound mutation must carry a UUID nonce");
assert!(!nonce.is_nil());
assert!(
crate::cluster::rpc::verify_tonic_rpc_signature(
audience,
"/node_service.NodeService/WriteAll",
request.metadata().as_ref(),
)
.is_ok()
);
}
#[test]
fn missing_capability_after_pin_fails_closed() {
ensure_test_rpc_secret();
let audience = "revoked-capability-client-test:9000";
let boot_epoch = Uuid::new_v4();
PEER_REPLAY_STATES
.lock()
.expect("peer capability cache lock must not be poisoned")
.insert(
audience.to_string(),
PeerReplayState {
boot_epoch: Some(boot_epoch),
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
},
);
let mut channel = ReplayScopeChannel::new(MissingProofService, Some(audience.to_string()));
futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
.expect("legacy response must complete before capability rejection");
let mut interceptor = test_interceptor_for(audience, false);
let error = interceptor
.call(rolling_mutation_request("Lock"))
.expect_err("a peer that loses its pinned capability must fail closed");
assert_eq!(error.code(), tonic::Code::Unauthenticated);
assert_eq!(error.message(), "RPC peer replay capability changed");
clear_peer_capability(audience);
}
#[tokio::test]
async fn tier_reload_recovers_only_with_authenticated_replay_capability() {
use super::super::peer_rest_client::{PeerRestClient, TierConfigReloadOutcome};
use bytes::Bytes;
use http_body_util::StreamBody;
use hyper::{body::Frame, service::service_fn};
use hyper_util::rt::{TokioExecutor, TokioIo};
use rustfs_protos::proto_gen::node_service::{LoadTransitionTierConfigResponse, PingResponse};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use tonic_prost::prost::Message;
ensure_test_rpc_secret();
let server_epoch = crate::cluster::rpc::http_auth::tonic_rpc_boot_epoch();
for (proof, restarted) in [
("valid", false),
("valid", true),
("missing", false),
("missing", true),
("invalid", false),
("invalid", true),
("rejected", false),
("rejected", true),
] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind test peer");
let address = listener.local_addr().expect("test peer address");
let audience = address.to_string();
let client = PeerRestClient::new(
rustfs_utils::XHost {
name: "127.0.0.1".to_string(),
port: address.port(),
is_port_set: true,
},
format!("http://{address}"),
);
let old_epoch = if restarted { Uuid::new_v4() } else { server_epoch };
let pinned = PeerReplayState {
boot_epoch: Some(old_epoch),
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch: old_epoch }),
};
set_peer_capability(&audience, pinned);
apply_peer_replay_response(audience.clone(), pinned, Err(std::io::Error::other("startup response omitted proof")));
let pings = Arc::new(AtomicUsize::new(0));
let reloads = Arc::new(AtomicUsize::new(0));
let server_audience = audience.clone();
let server_pings = pings.clone();
let server_reloads = reloads.clone();
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.expect("accept test client");
let service = service_fn(move |request: HttpRequest<hyper::body::Incoming>| {
let audience = server_audience.clone();
let pings = server_pings.clone();
let reloads = server_reloads.clone();
async move {
let authentication = crate::cluster::rpc::verify_tonic_rpc_signature_with_bootstrap(
&audience,
request.uri().path(),
request.headers(),
request.uri().path() == "/node_service.NodeService/Ping",
);
let stale_epoch = authentication
.as_ref()
.is_err_and(|error| error.to_string() == "RPC boot epoch is stale");
assert!(
authentication.is_ok() || stale_epoch,
"requests must retain valid signatures: {authentication:?}"
);
let challenge = tonic_boot_epoch_challenge(request.headers())
.expect("valid challenge")
.expect("challenge present");
let payload = match request.uri().path() {
"/node_service.NodeService/Ping" => {
pings.fetch_add(1, Ordering::SeqCst);
PingResponse {
version: 1,
body: Bytes::new(),
}
.encode_to_vec()
}
"/node_service.NodeService/LoadTransitionTierConfig" => {
reloads.fetch_add(1, Ordering::SeqCst);
assert!(
request.headers().contains_key(RPC_CONTENT_SHA256_HEADER),
"recovery must not downgrade body binding"
);
LoadTransitionTierConfigResponse {
success: true,
error_info: None,
error_code: None,
}
.encode_to_vec()
}
path => panic!("unexpected test RPC: {path}"),
};
let mut data = vec![0];
data.extend_from_slice(&u32::try_from(payload.len()).expect("small test protobuf").to_be_bytes());
data.extend_from_slice(&payload);
let mut trailers = http::HeaderMap::new();
trailers.insert(
"grpc-status",
if proof == "rejected" {
"16"
} else if stale_epoch {
"14"
} else {
"0"
}
.parse()
.expect("status header"),
);
if proof == "rejected" {
trailers.insert("grpc-message", "invalid credentials".parse().expect("message header"));
}
let frames: Vec<Result<Frame<Bytes>, std::convert::Infallible>> =
vec![Ok(Frame::data(Bytes::from(data))), Ok(Frame::trailers(trailers))];
let mut response = HttpResponse::new(StreamBody::new(futures::stream::iter(frames)));
response
.headers_mut()
.insert("content-type", "application/grpc".parse().expect("content type"));
if matches!(proof, "valid" | "invalid") {
let mut headers =
tonic_boot_epoch_response_headers(&audience, challenge).expect("sign response proof");
if proof == "invalid" {
headers.insert(
RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
"forged".parse().expect("forged proof header"),
);
}
response.headers_mut().extend(headers);
}
Ok::<_, std::convert::Infallible>(response)
}
});
hyper::server::conn::http2::Builder::new(TokioExecutor::new())
.serve_connection(TokioIo::new(stream), service)
.await
.expect("serve test peer");
});
let first =
tokio::time::timeout(std::time::Duration::from_secs(10), client.load_transition_tier_config_once_outcome())
.await
.expect("recovery attempt must finish");
assert_eq!(reloads.load(Ordering::SeqCst), 0, "revoked mutation must never reach the peer");
assert_eq!(
pings.load(Ordering::SeqCst),
1,
"revoked capability must trigger a signed read-only probe"
);
if proof == "rejected" {
assert!(
matches!(first, TierConfigReloadOutcome::Terminal(_)),
"invalid credentials must stay terminal"
);
} else {
assert!(
matches!(
first,
TierConfigReloadOutcome::TransientReconnect(_) | TierConfigReloadOutcome::TransientRetrySameChannel(_)
),
"a probe response must schedule another guarded reload"
);
let second =
tokio::time::timeout(std::time::Duration::from_secs(10), client.load_transition_tier_config_once_outcome())
.await
.expect("second reload attempt must finish");
if proof == "valid" {
assert!(matches!(second, TierConfigReloadOutcome::Success));
assert_eq!(reloads.load(Ordering::SeqCst), 1);
assert_eq!(pings.load(Ordering::SeqCst), 1);
assert_eq!(peer_replay_state(&audience).boot_epoch, Some(server_epoch));
} else {
assert!(matches!(
second,
TierConfigReloadOutcome::TransientReconnect(_) | TierConfigReloadOutcome::TransientRetrySameChannel(_)
));
assert_eq!(reloads.load(Ordering::SeqCst), 0, "unverified proof must not admit a mutation");
assert_eq!(pings.load(Ordering::SeqCst), 2);
assert_eq!(peer_replay_state(&audience).cache_capability, Some(PeerReplayCapability::Revoked));
}
}
server.abort();
let _ = server.await;
clear_peer_capability(&audience);
}
}
#[test]
fn test_signature_interceptor_binds_audience_from_peer_uri() {
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor())
.with_rpc_audience("http://node-a:9000")
.expect("peer URI should provide an audience");
let TonicInterceptor::Signature(interceptor) = interceptor else {
panic!("signature interceptor variant should be preserved");
};
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
}
#[test]
fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
ensure_test_rpc_secret();
let audience = "replay-scope-client-test:9000";
clear_peer_capability(audience);
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
let service = EpochProofService {
audience: audience.to_string(),
include_capability: true,
seen_headers: seen_headers.clone(),
};
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
let make_request = || replay_scope_request(audience, "Ping");
futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
assert_eq!(headers.len(), 2);
assert!(headers[0].contains_key(RPC_BOOT_EPOCH_CHALLENGE_HEADER));
assert!(
!headers[0].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
"the first request must remain v2-compatible until the peer proves its epoch"
);
assert!(
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
"the second request must carry the replay-scoped v3 signature"
);
clear_peer_capability(audience);
}
#[test]
fn test_signature_interceptor_requires_generated_method_metadata() {
ensure_test_rpc_secret();
let mut interceptor = test_interceptor();
let error = interceptor
.call(tonic::Request::new(()))
.expect_err("requests without an exact generated method must fail closed");
assert_eq!(error.code(), tonic::Code::Unauthenticated);
assert_eq!(error.message(), "Missing gRPC method metadata");
}
#[test]
fn test_signature_interceptor_may_inject_request_id() {
ensure_test_rpc_secret();
let mut interceptor = test_interceptor();
let req = test_request();
let span = tracing::info_span!("grpc-rpc-test-span");
let _guard = span.enter();
let req = interceptor.call(req).expect("interceptor call should succeed");
if let Some(v) = req.metadata().get("x-request-id") {
assert!(!v.as_encoded_bytes().is_empty());
}
}
#[test]
fn test_signature_interceptor_injects_traceparent_metadata() {
ensure_test_rpc_secret();
let mut interceptor = test_interceptor();
let req = test_request();
with_trace_parent("4bf92f3577b34da6a3ce929d0e0e4736", || {
let req = interceptor.call(req).expect("interceptor call should succeed");
let traceparent = req
.metadata()
.get("traceparent")
.and_then(|v| v.to_str().ok())
.expect("traceparent metadata should be injected");
assert!(traceparent.starts_with("00-4bf92f3577b34da6a3ce929d0e0e4736-"));
});
}
}