use std::num::NonZeroUsize;
use std::sync::Arc;
#[cfg(feature = "testing")]
use std::sync::Mutex;
use fred::clients::{Client, Pool};
use fred::interfaces::{ClientLike, EventInterface, PubsubInterface, StreamsInterface};
#[cfg(feature = "credential-provider")]
use fred::types::config::CredentialProvider;
#[cfg(any(
feature = "tls-rustls",
feature = "tls-rustls-ring",
feature = "tls-native-tls"
))]
use fred::types::config::TlsConfig;
use fred::types::config::{Config, PerformanceConfig, ReconnectPolicy, ServerConfig};
use fred::types::{ClientState, ClusterHash, CustomCommand, Message, Value, Version};
use fred::util::redis_keyslot;
#[cfg(feature = "testing")]
use ruststream::testing::{Coordinator, InProcess, TestableBroker};
use ruststream::{AddressedCopies, Broker, ConnectedBroker, DescribeServer, ServerSpec, Subscribe};
#[cfg(feature = "testing")]
use ruststream::{OutgoingMessage, RawMessage};
use tokio::runtime::Handle;
use tokio::sync::broadcast::Receiver;
use crate::address::{Address, redacted};
use crate::connection::{Closing, Connection, ReadConnection};
#[cfg(feature = "testing")]
use crate::convert::{fields_for_publish, parts_from_fields};
use crate::delay::DelayConfig;
#[cfg(feature = "testing")]
use crate::envelope::{frame, unframe};
#[cfg(feature = "testing")]
use crate::in_process::{self, Subscription, Write};
use crate::loopback::{Loopback, Tap};
use crate::recovery::RecoveryConfig;
use crate::{
error::RedisError,
list::{ListWire, RedisList, RedisListPublish, RedisListPublisher, RedisListSubscriber},
pipeline::{Owner, Rounds},
publisher::{RedisDefaultPublisher, RedisPublisher},
pubsub::{
PubSubMode, PubSubWire, RedisPubSub, RedisPubSubPattern, RedisPubSubPublish,
RedisPubSubPublisher, RedisPubSubSubscriber, Resubscribe, Resubscribing,
},
route::{Recorded, Route, Routes},
stream::{ReadMode, RedisStream},
subscriber::RedisSubscriber,
};
const DEFAULT_POOL_SIZE: usize = 4;
fn reconnect_policy() -> ReconnectPolicy {
ReconnectPolicy::new_exponential(0, 100, 10_000, 2)
}
const CLAIM_MIN_VERSION: Version = Version::new(8, 4, 0);
#[derive(Debug, Clone)]
enum Topology {
Standalone(Address),
Cluster(Vec<Address>),
Sentinel {
service: String,
hosts: Vec<Address>,
},
Preconnected(Pool),
}
fn parse_server(addr: &str, default_port: u16) -> Result<(String, u16), RedisError> {
let trimmed = addr
.trim()
.trim_start_matches("rediss://")
.trim_start_matches("redis://");
let (host, port) = match trimmed.rsplit_once(':') {
Some((host, port)) => {
let port = port.parse::<u16>().map_err(|_| {
let addr = redacted(addr);
RedisError::Connect(format!("invalid port in redis address `{addr}`").into())
})?;
(host, port)
}
None => (trimmed, default_port),
};
if host.is_empty() {
return Err(RedisError::Connect(
format!("missing host in redis address `{}`", redacted(addr)).into(),
));
}
Ok((host.to_owned(), port))
}
fn parse_servers(addrs: &[Address], default_port: u16) -> Result<Vec<(String, u16)>, RedisError> {
if addrs.is_empty() {
return Err(RedisError::Connect("no redis addresses provided".into()));
}
addrs
.iter()
.map(|addr| parse_server(addr, default_port))
.collect()
}
#[derive(Clone, Default)]
struct AuthConfig {
username: Option<String>,
password: Option<String>,
#[cfg(feature = "sentinel-auth")]
sentinel_username: Option<String>,
#[cfg(feature = "sentinel-auth")]
sentinel_password: Option<String>,
#[cfg(any(
feature = "tls-rustls",
feature = "tls-rustls-ring",
feature = "tls-native-tls"
))]
tls: Option<TlsConfig>,
#[cfg(feature = "credential-provider")]
credential_provider: Option<Arc<dyn CredentialProvider>>,
}
impl std::fmt::Debug for AuthConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut s = f.debug_struct("AuthConfig");
s.field("username", &self.username);
s.field("password", &self.password.as_ref().map(|_| "<redacted>"));
#[cfg(feature = "sentinel-auth")]
{
s.field("sentinel_username", &self.sentinel_username);
s.field(
"sentinel_password",
&self.sentinel_password.as_ref().map(|_| "<redacted>"),
);
}
#[cfg(any(
feature = "tls-rustls",
feature = "tls-rustls-ring",
feature = "tls-native-tls"
))]
s.field("tls", &self.tls.as_ref().map(|_| "<configured>"));
#[cfg(feature = "credential-provider")]
s.field(
"credential_provider",
&self.credential_provider.as_ref().map(|_| "<configured>"),
);
s.finish()
}
}
#[derive(Debug, Clone)]
#[must_use]
pub struct RedisBroker {
topology: Topology,
pool_size: usize,
default_group: Option<String>,
auth: AuthConfig,
}
impl RedisBroker {
pub fn standalone(url: impl Into<String>) -> Self {
Self::with_topology(Topology::Standalone(Address::new(url.into())))
}
pub fn cluster(nodes: impl IntoIterator<Item = impl Into<String>>) -> Self {
Self::with_topology(Topology::Cluster(
nodes
.into_iter()
.map(|node| Address::new(node.into()))
.collect(),
))
}
pub fn sentinel(
service: impl Into<String>,
sentinels: impl IntoIterator<Item = impl Into<String>>,
) -> Self {
Self::with_topology(Topology::Sentinel {
service: service.into(),
hosts: sentinels
.into_iter()
.map(|host| Address::new(host.into()))
.collect(),
})
}
pub fn from_pool(pool: Pool) -> Self {
Self::with_topology(Topology::Preconnected(pool))
}
fn with_topology(topology: Topology) -> Self {
Self {
topology,
pool_size: DEFAULT_POOL_SIZE,
default_group: None,
auth: AuthConfig::default(),
}
}
pub const fn pool(mut self, size: usize) -> Self {
self.pool_size = size;
self
}
pub fn default_group(mut self, group: impl Into<String>) -> Self {
self.default_group = Some(group.into());
self
}
pub fn credentials(mut self, username: impl Into<String>, password: impl Into<String>) -> Self {
self.auth.username = Some(username.into());
self.auth.password = Some(password.into());
self
}
pub fn password(mut self, password: impl Into<String>) -> Self {
self.auth.password = Some(password.into());
self
}
#[cfg(any(
feature = "tls-rustls",
feature = "tls-rustls-ring",
feature = "tls-native-tls"
))]
pub fn tls(mut self, tls: impl Into<TlsConfig>) -> Self {
self.auth.tls = Some(tls.into());
self
}
#[cfg(feature = "sentinel-auth")]
pub fn sentinel_credentials(
mut self,
username: impl Into<String>,
password: impl Into<String>,
) -> Self {
self.auth.sentinel_username = Some(username.into());
self.auth.sentinel_password = Some(password.into());
self
}
#[cfg(feature = "sentinel-auth")]
pub fn sentinel_password(mut self, password: impl Into<String>) -> Self {
self.auth.sentinel_password = Some(password.into());
self
}
#[cfg(feature = "credential-provider")]
pub fn credential_provider(mut self, provider: Arc<dyn CredentialProvider>) -> Self {
self.auth.credential_provider = Some(provider);
self
}
fn build_config(&self) -> Result<Config, RedisError> {
let mut config = match &self.topology {
Topology::Standalone(url) => {
Config::from_url(url).map_err(|err| RedisError::Connect(Box::new(err)))?
}
Topology::Cluster(nodes) => {
let hosts = parse_servers(nodes, 6379)?;
Config {
server: ServerConfig::new_clustered(hosts),
..Config::default()
}
}
Topology::Sentinel { service, hosts } => {
let hosts = parse_servers(hosts, 26379)?;
Config {
server: ServerConfig::new_sentinel(hosts, service.clone()),
..Config::default()
}
}
Topology::Preconnected(pool) => pool.next().client_config(),
};
self.apply_auth(&mut config);
Ok(config)
}
fn apply_auth(&self, config: &mut Config) {
if self.auth.username.is_some() {
config.username.clone_from(&self.auth.username);
}
if self.auth.password.is_some() {
config.password.clone_from(&self.auth.password);
}
#[cfg(any(
feature = "tls-rustls",
feature = "tls-rustls-ring",
feature = "tls-native-tls"
))]
if self.auth.tls.is_some() {
config.tls.clone_from(&self.auth.tls);
}
#[cfg(feature = "credential-provider")]
if self.auth.credential_provider.is_some() {
config
.credential_provider
.clone_from(&self.auth.credential_provider);
}
#[cfg(feature = "sentinel-auth")]
if let ServerConfig::Sentinel {
username, password, ..
} = &mut config.server
{
if self.auth.sentinel_username.is_some() {
username.clone_from(&self.auth.sentinel_username);
}
if self.auth.sentinel_password.is_some() {
password.clone_from(&self.auth.sentinel_password);
}
}
}
}
impl Broker for RedisBroker {
type Error = RedisError;
type Connected = ConnectedRedisBroker;
async fn connect(self) -> Result<Self::Connected, Self::Error> {
let config = self.build_config()?;
let clustered = config.server.is_clustered();
let (pool, reconnect) = if let Topology::Preconnected(pool) = &self.topology {
let started = pool
.clients()
.iter()
.any(|client| client.state() != ClientState::Disconnected);
if !started {
pool.init()
.await
.map_err(|err| RedisError::Connect(Box::new(err)))?;
}
(pool.clone(), pool.next().client_reconnect_policy())
} else {
let reconnect = reconnect_policy();
let pool = Pool::new(
config.clone(),
None,
None,
Some(reconnect.clone()),
self.pool_size,
)
.map_err(|err| RedisError::Connect(Box::new(err)))?;
pool.init()
.await
.map_err(|err| RedisError::Connect(Box::new(err)))?;
(pool, Some(reconnect))
};
Ok(ConnectedRedisBroker {
core: Arc::new(RedisCore {
connection: Connection::new(pool),
closing: Arc::default(),
runtime: Handle::current(),
config,
reconnect,
default_group: self.default_group,
clustered,
routes: Routes::default(),
rounds: Arc::default(),
loopback: Loopback::default(),
#[cfg(feature = "testing")]
opened: Mutex::default(),
}),
})
}
}
#[cfg(feature = "testing")]
impl InProcess for RedisBroker {
async fn connect_in_process(self) -> Result<Self::Connected, Self::Error> {
let config = self.build_config()?;
let clustered = config.server.is_clustered();
let pool_size = match &self.topology {
Topology::Preconnected(pool) => pool.size(),
_ => self.pool_size,
};
let (pool, config, server) = in_process::connect(&config, pool_size, clustered).await?;
Ok(ConnectedRedisBroker {
core: Arc::new(RedisCore {
connection: Connection::new(pool),
closing: Arc::default(),
runtime: Handle::current(),
config,
reconnect: None,
default_group: self.default_group,
clustered,
routes: Routes::default(),
rounds: Arc::default(),
loopback: Loopback::to(server),
opened: Mutex::default(),
}),
})
}
}
#[cfg(feature = "testing")]
ruststream::register_testable_broker!(RedisBroker);
impl DescribeServer for RedisBroker {
fn describe_server(&self) -> ServerSpec {
let host = match &self.topology {
Topology::Standalone(url) => describe_address(url, 6379),
Topology::Cluster(nodes) => nodes
.first()
.map_or_else(String::new, |node| describe_address(node, 6379)),
Topology::Sentinel { hosts, .. } => hosts
.first()
.map_or_else(String::new, |host| describe_address(host, 26379)),
Topology::Preconnected(_) => String::new(),
};
let mut spec = ServerSpec::new(host, "redis");
if spec.host.as_deref() == Some("") {
spec.host = None;
}
spec
}
}
fn describe_address(addr: &str, default_port: u16) -> String {
let host = ServerSpec::host_from_url(addr.trim());
if host.is_empty() {
return String::new();
}
if has_port(&host) {
host
} else {
format!("{host}:{default_port}")
}
}
fn has_port(host: &str) -> bool {
host.rsplit_once(']').map_or_else(
|| {
host.rsplit_once(':')
.is_some_and(|(_, port)| !port.is_empty())
},
|(_, after_bracket)| after_bracket.starts_with(':'),
)
}
pub(crate) struct RedisCore {
connection: Arc<Connection>,
closing: Arc<Closing>,
runtime: Handle,
config: Config,
reconnect: Option<ReconnectPolicy>,
default_group: Option<String>,
clustered: bool,
routes: Routes,
rounds: Arc<Rounds>,
loopback: Loopback,
#[cfg(feature = "testing")]
opened: Mutex<Vec<Opened>>,
}
#[cfg(feature = "testing")]
#[derive(Debug, Clone)]
enum Opened {
Stream { key: String, group: String },
List { key: String },
Channel { name: String, sharded: bool },
Pattern { glob: String },
}
impl RedisCore {
pub(crate) fn rounds(&self) -> &Arc<Rounds> {
&self.rounds
}
pub(crate) const fn routes(&self) -> &Routes {
&self.routes
}
pub(crate) fn record_routes(
&self,
name: &str,
dead_letter: Option<&str>,
route: &Route,
) -> Result<Recorded<'_>, RedisError> {
self.routes.record_subscription(name, dead_letter, route)
}
pub(crate) fn pool(&self) -> Result<Pool, RedisError> {
self.connection.live().cloned()
}
pub(crate) fn connection(&self) -> Result<Arc<Connection>, RedisError> {
self.connection.live()?;
Ok(Arc::clone(&self.connection))
}
pub(crate) const fn transactions_supported(&self) -> bool {
!self.clustered
}
pub(crate) const fn clustered(&self) -> bool {
self.clustered
}
pub(crate) const fn loopback(&self) -> &Loopback {
&self.loopback
}
#[cfg(feature = "testing")]
fn opened(&self, subscription: Opened) {
self.opened
.lock()
.expect("redis subscription record poisoned")
.push(subscription);
}
#[cfg_attr(
not(feature = "testing"),
allow(
clippy::unused_self,
reason = "the connection is read under `testing` only"
)
)]
fn server_version(&self, pool: &Pool) -> Option<Version> {
#[cfg(feature = "testing")]
if self.loopback.server().is_some() {
return Some(in_process::VERSION);
}
pool.server_version()
}
}
impl std::fmt::Debug for RedisCore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RedisCore")
.field("pool", self.connection.pool())
.field("closed", &self.connection.is_closed())
.finish_non_exhaustive()
}
}
#[derive(Debug)]
pub struct ConnectedRedisBroker {
core: Arc<RedisCore>,
}
impl ConnectedRedisBroker {
pub async fn subscribe(&self, def: RedisStream) -> Result<RedisSubscriber, RedisError> {
let connection = self.core.connection()?;
let pool = connection.pool();
let group = def.group_or_err()?.to_owned();
let consumer = def.consumer_or_auto();
if let ReadMode::Claiming { .. } = def.mode() {
require_claim_support(self.core.server_version(pool).as_ref(), def.key())?;
}
let recorded = self
.core
.record_routes(def.key(), def.dead_letter(), &Route::Stream)?;
ensure_group(pool, def.key(), &group, def.start().as_id()).await?;
let reader = self.read_connection().await?;
let delay = def.delay_config().map(|cfg| cfg.on(self.core.loopback()));
let tap = self.stream_tap(&def, &group, &consumer, delay.as_ref());
recorded.keep();
Ok(RedisSubscriber::new(
connection,
reader,
def.key().to_owned(),
group,
consumer,
def.block_or_default(),
def.mode(),
delay,
tap,
))
}
#[cfg_attr(
not(feature = "testing"),
allow(
clippy::unused_self,
reason = "the connection is read under `testing` only"
)
)]
fn stream_tap(
&self,
def: &RedisStream,
group: &str,
consumer: &str,
delay: Option<&DelayConfig>,
) -> Tap {
#[cfg(feature = "testing")]
{
self.core.opened(Opened::Stream {
key: def.key().to_owned(),
group: group.to_owned(),
});
if let Some(server) = self.core.loopback().server() {
let (claim, fresh) = match def.mode() {
ReadMode::Fresh => (None, true),
ReadMode::Reclaim { min_idle } => (Some(min_idle), false),
ReadMode::Claiming { min_idle } => (Some(min_idle), true),
};
let reader = server.attach_stream(
def.key(),
group,
consumer,
claim,
fresh,
delay.map(DelayConfig::zset_key),
);
return Tap::registered(Arc::clone(server), reader);
}
Tap::default()
}
#[cfg(not(feature = "testing"))]
{
let _ = (def, group, consumer, delay);
Tap::default()
}
}
pub async fn subscribe_pubsub(
&self,
def: RedisPubSub,
) -> Result<RedisPubSubSubscriber, RedisError> {
Ok(RedisPubSubSubscriber::new(self.open_pubsub(def).await?))
}
pub(crate) async fn open_pubsub(&self, def: RedisPubSub) -> Result<PubSubWire, RedisError> {
let codec = def.codec_handle();
let recorded = self
.core
.record_routes(def.channel(), def.dead_letter(), &def.route())?;
let client = self.new_client(def.buffer_size()).await?;
let rx = client.message_rx();
let reconnects = client.reconnect_rx();
let channel = def.channel().to_owned();
let sharded = def.delivery_mode() == PubSubMode::Sharded;
let target = if sharded {
Resubscribe::ShardedChannel(channel.clone())
} else {
Resubscribe::Channel(channel.clone())
};
match def.delivery_mode() {
PubSubMode::Classic => {
client
.subscribe(channel)
.await
.map_err(RedisError::subscribe)?;
confirm_subscribed(&client).await?;
}
PubSubMode::Sharded => {
client
.ssubscribe(channel)
.await
.map_err(RedisError::subscribe)?;
}
}
let (rx, tap) = self.pubsub_tap(
rx,
PubSubTarget::Channel {
name: def.channel(),
sharded,
},
def.buffer_size(),
);
let pool = self.core.pool()?;
recorded.keep();
let client = Resubscribing::new(client, reconnects, target, self.runtime());
Ok(PubSubWire::new(
def.channel().to_owned(),
client,
rx,
codec,
pool,
tap,
self.runtime().clone(),
))
}
#[cfg_attr(
not(feature = "testing"),
allow(
clippy::unused_self,
reason = "the connection is read under `testing` only"
)
)]
fn pubsub_tap(
&self,
rx: Receiver<Message>,
target: PubSubTarget<'_>,
buffer: Option<NonZeroUsize>,
) -> (Receiver<Message>, Tap) {
#[cfg(feature = "testing")]
{
let (opened, subscription) = match target {
PubSubTarget::Channel { name, sharded } => {
let bytes = bytes::Bytes::copy_from_slice(name.as_bytes());
(
Opened::Channel {
name: name.to_owned(),
sharded,
},
if sharded {
Subscription::Sharded(bytes)
} else {
Subscription::Channel(bytes)
},
)
}
PubSubTarget::Pattern(glob) => (
Opened::Pattern {
glob: glob.to_owned(),
},
Subscription::Pattern(bytes::Bytes::copy_from_slice(glob.as_bytes())),
),
};
self.core.opened(opened);
if let Some(server) = self.core.loopback().server() {
let (rx, reader) = server.attach_pubsub(subscription, buffer);
return (rx, Tap::registered(Arc::clone(server), reader));
}
(rx, Tap::default())
}
#[cfg(not(feature = "testing"))]
{
let _ = (target, buffer);
(rx, Tap::default())
}
}
pub async fn subscribe_pubsub_pattern(
&self,
def: RedisPubSubPattern,
) -> Result<RedisPubSubSubscriber, RedisError> {
Ok(RedisPubSubSubscriber::new(
self.open_pubsub_pattern(def).await?,
))
}
pub(crate) async fn open_pubsub_pattern(
&self,
def: RedisPubSubPattern,
) -> Result<PubSubWire, RedisError> {
let codec = def.codec_handle();
let client = self.new_client(def.buffer_size()).await?;
let rx = client.message_rx();
let reconnects = client.reconnect_rx();
client
.psubscribe(def.pattern().to_owned())
.await
.map_err(RedisError::subscribe)?;
confirm_subscribed(&client).await?;
let (rx, tap) =
self.pubsub_tap(rx, PubSubTarget::Pattern(def.pattern()), def.buffer_size());
let target = Resubscribe::Pattern(def.pattern().to_owned());
let client = Resubscribing::new(client, reconnects, target, self.runtime());
Ok(PubSubWire::new(
def.pattern().to_owned(),
client,
rx,
codec,
self.core.pool()?,
tap,
self.runtime().clone(),
))
}
pub async fn subscribe_list(&self, def: RedisList) -> Result<RedisListSubscriber, RedisError> {
self.open_list(def).await.map(RedisListSubscriber::new)
}
pub(crate) async fn open_list(&self, def: RedisList) -> Result<ListWire, RedisError> {
let connection = self.core.connection()?;
let recovery = def
.recovery_config()?
.map(|cfg| cfg.on(self.core.loopback()));
let reliable = def.is_reliable();
let processing = def.processing_or_default();
if reliable && self.core.clustered() {
require_one_slot(def.key(), &processing)?;
}
self.core
.record_routes(def.key(), def.dead_letter(), &def.route())?
.keep();
let block = def.block_or_default();
let codec = def.codec_handle();
let reader = self.read_connection().await?;
let tap = self.list_tap(def.key(), recovery.as_ref());
Ok(ListWire::new(
connection,
reader,
def.into_key(),
reliable,
processing,
block,
codec,
recovery,
tap,
))
}
#[cfg_attr(
not(feature = "testing"),
allow(
clippy::unused_self,
reason = "the connection is read under `testing` only"
)
)]
fn list_tap(&self, key: &str, recovery: Option<&RecoveryConfig>) -> Tap {
#[cfg(feature = "testing")]
{
self.core.opened(Opened::List {
key: key.to_owned(),
});
if let Some(server) = self.core.loopback().server() {
let reader = server.attach_list(
key,
recovery.map(|cfg| (cfg.zset_key.as_str(), cfg.min_idle)),
);
return Tap::registered(Arc::clone(server), reader);
}
Tap::default()
}
#[cfg(not(feature = "testing"))]
{
let _ = (key, recovery);
Tap::default()
}
}
#[must_use]
pub fn publisher(&self) -> RedisPublisher {
RedisPublisher::new(Arc::clone(&self.core))
}
#[must_use]
pub fn default_publisher(&self) -> RedisDefaultPublisher {
RedisDefaultPublisher::new(Arc::clone(&self.core))
}
#[must_use]
pub fn pubsub_publisher(&self, publish: RedisPubSubPublish) -> RedisPubSubPublisher {
RedisPubSubPublisher::new(Arc::clone(&self.core), publish)
}
#[must_use]
pub fn list_publisher(&self, publish: RedisListPublish) -> RedisListPublisher {
RedisListPublisher::new(Arc::clone(&self.core), publish)
}
pub(crate) fn rounds(&self) -> &Arc<Rounds> {
self.core.rounds()
}
pub(crate) fn runtime(&self) -> &Handle {
&self.core.runtime
}
pub(crate) fn window_owner(&self) -> Result<Owner, RedisError> {
Ok(Owner {
rounds: Arc::clone(self.rounds()),
runtime: self.runtime().clone(),
connection: self.core.connection()?,
})
}
pub fn pool_handle(&self) -> Result<Pool, RedisError> {
self.core.pool()
}
async fn read_connection(&self) -> Result<ReadConnection, RedisError> {
#[cfg(feature = "testing")]
let blocks = self.core.loopback().server().is_none();
#[cfg(not(feature = "testing"))]
let blocks = true;
Ok(ReadConnection::new(
self.new_client(None).await?,
self.runtime().clone(),
Arc::clone(&self.core.closing),
blocks,
))
}
async fn new_client(&self, buffer: Option<NonZeroUsize>) -> Result<Client, RedisError> {
let _ = self.core.pool()?;
let performance = buffer.map(|buffer| PerformanceConfig {
broadcast_channel_capacity: buffer.get(),
..PerformanceConfig::default()
});
let client = Client::new(
self.core.config.clone(),
performance,
None,
self.core.reconnect.clone(),
);
let connecting = client.clone();
self.core
.runtime
.spawn(async move { connecting.init().await })
.await
.map_err(|err| RedisError::Connect(Box::new(err)))?
.map_err(|err| RedisError::Connect(Box::new(err)))?;
Ok(client)
}
}
impl ConnectedBroker for ConnectedRedisBroker {
type Error = RedisError;
type Closed = ClosedRedisBroker;
async fn shutdown(self) -> Result<Self::Closed, Self::Error> {
self.core.connection.close();
self.core.closing.finish().await;
let connections_closed = self.core.connection.pool().clients().len();
self.core
.connection
.pool()
.quit()
.await
.map_err(|err| RedisError::Connect(Box::new(err)))?;
#[cfg(feature = "testing")]
if let Some(server) = self.core.loopback().server() {
server.close();
}
Ok(ClosedRedisBroker { connections_closed })
}
}
#[cfg_attr(
not(feature = "testing"),
allow(dead_code, reason = "read by the in-process registration only")
)]
#[derive(Clone, Copy)]
enum PubSubTarget<'a> {
Channel { name: &'a str, sharded: bool },
Pattern(&'a str),
}
#[cfg(feature = "testing")]
impl TestableBroker for ConnectedRedisBroker {
fn install_coordinator(&self, coordinator: Coordinator) {
if let Some(server) = self.core.loopback().server() {
server.install(coordinator);
}
}
fn inject(&self, message: OutgoingMessage<'_>) {
let server = self.in_process("inject");
let name = message.name();
let (payload, headers) = (message.payload(), message.headers());
let written = match self.core.routes().route(name) {
Route::Stream if self.read_by_patterns_alone(name) => {
server.inject_publish(name, frame(None, payload, headers), false)
}
Route::Stream => {
server.inject_stream(name, fields_for_publish(payload.to_vec(), headers))
}
Route::List { envelope } => {
server.inject_list(name, frame(envelope.as_ref(), payload, headers))
}
Route::Channel { mode, envelope } => server.inject_publish(
name,
frame(envelope.as_ref(), payload, headers),
mode == PubSubMode::Sharded,
),
};
if let Err(err) = written {
panic!("the injected message to {name:?} is not one Redis takes: {err}");
}
}
fn published(&self, name: &str) -> Vec<RawMessage> {
let server = self.in_process("published");
let envelope = match self.core.routes().route(name) {
Route::Stream => None,
Route::List { envelope } | Route::Channel { envelope, .. } => envelope,
};
server
.published(name)
.into_iter()
.map(|write| {
let (payload, headers) = match write {
Write::Fields(fields) => parts_from_fields(
fields
.into_iter()
.map(|(field, value)| {
(String::from_utf8_lossy(&field).into_owned(), value.to_vec())
})
.collect(),
),
Write::Body(body) => unframe(envelope.as_ref(), &body),
};
RawMessage::new(name.to_owned(), payload).with_headers(headers)
})
.collect()
}
fn routes(&self, destination: &str, subscriptions: &[&str]) -> Vec<usize> {
let opened = self
.core
.opened
.lock()
.expect("redis subscription record poisoned")
.clone();
let mut owed = Vec::new();
let mut owe = |name: &str, times: usize| {
if let Some(at) = subscriptions.iter().position(|held| *held == name) {
owed.extend(std::iter::repeat_n(at, times));
}
};
let groups: std::collections::BTreeSet<&str> = opened
.iter()
.filter_map(|sub| match sub {
Opened::Stream { key, group } if key == destination => Some(group.as_str()),
_ => None,
})
.collect();
let patterns = match self.core.routes().route(destination) {
Route::Stream => {
owe(destination, groups.len());
groups.is_empty()
}
Route::List { .. } => {
let read = opened
.iter()
.any(|sub| matches!(sub, Opened::List { key } if key == destination));
owe(destination, usize::from(read));
false
}
Route::Channel { mode, .. } => {
let sharded = mode == PubSubMode::Sharded;
let reached = opened
.iter()
.filter(|sub| {
matches!(sub, Opened::Channel { name, sharded: held }
if name == destination && *held == sharded)
})
.count();
owe(destination, reached);
!sharded
}
};
if patterns {
for sub in &opened {
if let Opened::Pattern { glob } = sub
&& in_process::glob_matches(glob.as_bytes(), destination.as_bytes())
{
owe(glob, 1);
}
}
}
owed
}
}
#[cfg(feature = "testing")]
impl ConnectedRedisBroker {
fn read_by_patterns_alone(&self, name: &str) -> bool {
let opened = self
.core
.opened
.lock()
.expect("redis subscription record poisoned");
let grouped = opened
.iter()
.any(|sub| matches!(sub, Opened::Stream { key, .. } if key == name));
!grouped
&& opened.iter().any(|sub| {
matches!(sub, Opened::Pattern { glob }
if in_process::glob_matches(glob.as_bytes(), name.as_bytes()))
})
}
fn in_process(&self, what: &str) -> &Arc<in_process::Server> {
self.core.loopback().server().unwrap_or_else(|| {
panic!(
"TestableBroker::{what} reached a broker connected with `connect`; the harness \
drives the connection `connect_in_process` produces"
)
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ClosedRedisBroker {
connections_closed: usize,
}
impl ClosedRedisBroker {
#[must_use]
pub const fn connections_closed(&self) -> usize {
self.connections_closed
}
}
#[allow(
clippy::use_self,
reason = "the type name disambiguates the inherent subscribe from this trait method"
)]
impl Subscribe for ConnectedRedisBroker {
type Subscriber = RedisSubscriber;
type Copies = AddressedCopies;
async fn subscribe(&self, name: &str) -> Result<Self::Subscriber, Self::Error> {
let group = self
.core
.default_group
.clone()
.ok_or_else(|| missing_default_group(name, "RedisBroker"))?;
ConnectedRedisBroker::subscribe(self, RedisStream::new(name).group(group)).await
}
}
pub(crate) fn missing_default_group(name: &str, broker: &str) -> RedisError {
RedisError::InvalidOptions(format!(
"bare-string subscription on `{name}` needs a broker-wide default group: call \
{broker}::default_group(name), or subscribe with RedisStream::new(name).group(group)"
))
}
fn require_claim_support(version: Option<&Version>, key: &str) -> Result<(), RedisError> {
match version {
Some(version) if *version >= CLAIM_MIN_VERSION => Ok(()),
Some(version) => Err(RedisError::ServerTooOld(format!(
"RedisStream::claiming on {key:?} needs Redis {CLAIM_MIN_VERSION} or later \
(XREADGROUP CLAIM); the connected server is {version}"
))),
None => Err(RedisError::ServerTooOld(format!(
"RedisStream::claiming on {key:?} needs Redis {CLAIM_MIN_VERSION} or later \
(XREADGROUP CLAIM); the connected server reports no version"
))),
}
}
async fn confirm_subscribed(client: &Client) -> Result<(), RedisError> {
let node = ClusterHash::Custom(redis_keyslot(client.id().as_bytes()));
let _: Value = client
.custom(CustomCommand::new("PING", node, false), Vec::<Value>::new())
.await
.map_err(RedisError::subscribe)?;
Ok(())
}
fn require_one_slot(key: &str, processing: &str) -> Result<(), RedisError> {
if redis_keyslot(key.as_bytes()) == redis_keyslot(processing.as_bytes()) {
return Ok(());
}
Err(RedisError::InvalidOptions(format!(
"a reliable list on `{key}` cannot run on a cluster while its processing list \
`{processing}` hashes to another slot: a claim moves the entry between the two in one \
command. Put both on one slot with a hash tag, as in RedisList::new(\"{{{key}}}\"), which \
the default processing key follows"
)))
}
pub(crate) async fn ensure_group(
pool: &Pool,
key: &str,
group: &str,
start_id: &str,
) -> Result<(), RedisError> {
let result: Result<String, fred::error::Error> =
pool.xgroup_create(key, group, start_id, true).await;
match result {
Ok(_) => Ok(()),
Err(err) if err.details().contains("BUSYGROUP") => Ok(()),
Err(err) => Err(RedisError::subscribe(err)),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_claiming_subscription_names_both_versions_when_it_refuses() {
let err = require_claim_support(Some(&Version::new(7, 4, 2)), "orders")
.expect_err("a server below 8.4.0 must be refused");
assert_eq!(
err.to_string(),
"RedisStream::claiming on \"orders\" needs Redis 8.4.0 or later (XREADGROUP CLAIM); \
the connected server is 7.4.2"
);
}
#[test]
fn a_server_with_no_version_is_refused() {
let err = require_claim_support(None, "orders")
.expect_err("a server reporting no version must be refused");
assert!(
err.to_string().contains("reports no version"),
"the refusal has to say the version is missing: {err}"
);
}
#[test]
fn the_floor_release_and_later_are_accepted() {
for version in [
Version::new(8, 4, 0),
Version::new(8, 4, 6),
Version::new(9, 0, 0),
] {
assert!(
require_claim_support(Some(&version), "orders").is_ok(),
"{version} has XREADGROUP CLAIM"
);
}
for version in [Version::new(8, 3, 9), Version::new(7, 4, 2)] {
assert!(
require_claim_support(Some(&version), "orders").is_err(),
"{version} does not have XREADGROUP CLAIM"
);
}
}
#[test]
fn describe_server_reports_redis() {
let broker = RedisBroker::standalone("redis://localhost:6379");
let spec = broker.describe_server();
assert_eq!(spec.protocol, "redis");
assert_eq!(spec.host.as_deref(), Some("localhost:6379"));
}
#[test]
fn a_description_is_the_host_and_port_of_every_address_shape() {
let cases = [
("redis://localhost:6379", "localhost:6379"),
("rediss://redis.example.com:6380", "redis.example.com:6380"),
("redis://localhost", "localhost:6379"),
("redis://localhost:6379/3", "localhost:6379"),
("redis://localhost:6379/?timeout=5s", "localhost:6379"),
(
"valkeys://valkey.example.com:6380",
"valkey.example.com:6380",
),
("10.0.0.1:6379", "10.0.0.1:6379"),
(" redis://localhost:6379 ", "localhost:6379"),
("redis://[::1]:6379", "[::1]:6379"),
("redis://[::1]", "[::1]:6379"),
];
for (url, expected) in cases {
let spec = RedisBroker::standalone(url).describe_server();
assert_eq!(
spec.host.as_deref(),
Some(expected),
"the description of {url:?} is wrong"
);
}
let cluster = RedisBroker::cluster(["10.0.0.1", "10.0.0.2:7001"]).describe_server();
assert_eq!(cluster.host.as_deref(), Some("10.0.0.1:6379"));
let sentinel = RedisBroker::sentinel("mymaster", ["10.0.0.9"]).describe_server();
assert_eq!(sentinel.host.as_deref(), Some("10.0.0.9:26379"));
}
#[test]
fn a_description_never_carries_the_credentials_of_a_url() {
let cases = [
(
RedisBroker::standalone("redis://alice:p@ss@redis.example.com:6379/0"),
"redis.example.com:6379",
),
(
RedisBroker::cluster(["redis://alice:p@ss@10.0.0.1:7000"]),
"10.0.0.1:7000",
),
(
RedisBroker::sentinel("mymaster", ["redis://alice:p@ss@10.0.0.9:26379"]),
"10.0.0.9:26379",
),
];
for (broker, expected) in cases {
let host = broker.describe_server().host.expect("a described host");
assert_eq!(host, expected, "the description kept more than the address");
assert!(
!host.contains('@'),
"{host:?} still carries a userinfo part"
);
assert!(
!host.contains("alice"),
"{host:?} still carries the user name"
);
assert!(
!host.contains("p@ss"),
"{host:?} still carries the password"
);
}
}
#[test]
fn credentials_apply_to_all_topologies() {
let brokers = [
RedisBroker::standalone("redis://localhost:6379").credentials("alice", "s3cr3t"),
RedisBroker::cluster(["127.0.0.1:7000"]).credentials("alice", "s3cr3t"),
RedisBroker::sentinel("mymaster", ["127.0.0.1:26379"]).credentials("alice", "s3cr3t"),
];
for broker in brokers {
let config = broker.build_config().expect("config builds");
assert_eq!(config.username.as_deref(), Some("alice"));
assert_eq!(config.password.as_deref(), Some("s3cr3t"));
}
}
#[test]
fn password_only_sets_password_without_username() {
let config = RedisBroker::cluster(["127.0.0.1:7000"])
.password("requirepass")
.build_config()
.expect("config builds");
assert_eq!(config.username, None);
assert_eq!(config.password.as_deref(), Some("requirepass"));
}
#[test]
fn programmatic_credentials_override_standalone_url() {
let config = RedisBroker::standalone("redis://urluser:urlpass@localhost:6379")
.credentials("acluser", "aclpass")
.build_config()
.expect("config builds");
assert_eq!(config.username.as_deref(), Some("acluser"));
assert_eq!(config.password.as_deref(), Some("aclpass"));
}
#[test]
fn url_credentials_preserved_without_override() {
let config = RedisBroker::standalone("redis://urluser:urlpass@localhost:6379")
.build_config()
.expect("config builds");
assert_eq!(config.username.as_deref(), Some("urluser"));
assert_eq!(config.password.as_deref(), Some("urlpass"));
}
#[test]
fn debug_redacts_password() {
let broker =
RedisBroker::standalone("redis://localhost:6379").credentials("alice", "s3cr3t");
let rendered = format!("{broker:?}");
assert!(
!rendered.contains("s3cr3t"),
"password must not appear in Debug output: {rendered}"
);
assert!(
rendered.contains("alice"),
"expected username in: {rendered}"
);
}
#[test]
fn an_address_error_names_no_password() {
for broker in [
RedisBroker::cluster(["redis://bob:seed-secret@node1:notaport"]),
RedisBroker::sentinel("mymaster", ["bob:seed-secret@sentinel1:notaport"]),
] {
let error = broker.build_config().expect_err("the address is invalid");
let printed = error.to_string();
assert!(printed.contains("bob"), "the address is named: {printed}");
assert!(
!printed.contains("seed-secret"),
"the password leaks: {printed}"
);
}
}
#[cfg(feature = "sentinel-auth")]
#[test]
fn sentinel_credentials_apply_to_sentinel_server() {
let config = RedisBroker::sentinel("mymaster", ["127.0.0.1:26379"])
.credentials("datauser", "datapass")
.sentinel_credentials("sentineluser", "sentinelpass")
.build_config()
.expect("config builds");
assert_eq!(config.username.as_deref(), Some("datauser"));
let ServerConfig::Sentinel {
username, password, ..
} = &config.server
else {
panic!("expected a sentinel server config");
};
assert_eq!(username.as_deref(), Some("sentineluser"));
assert_eq!(password.as_deref(), Some("sentinelpass"));
}
#[cfg(feature = "credential-provider")]
#[derive(Debug)]
struct StaticCredentials;
#[cfg(feature = "credential-provider")]
#[async_trait::async_trait]
impl CredentialProvider for StaticCredentials {
async fn fetch(
&self,
_server: Option<&fred::types::config::Server>,
) -> Result<(Option<String>, Option<String>), fred::error::Error> {
Ok((Some("rotating".into()), Some("token".into())))
}
}
#[cfg(feature = "credential-provider")]
#[test]
fn credential_provider_is_applied() {
let provider: Arc<dyn CredentialProvider> = Arc::new(StaticCredentials);
let config = RedisBroker::cluster(["127.0.0.1:7000"])
.credential_provider(provider)
.build_config()
.expect("config builds");
assert!(config.credential_provider.is_some());
}
}