sisa_caching_redis/connection.rs
1//! The connections a Redis store runs on: one server, a cluster, or a primary behind Sentinel.
2
3use std::{
4 collections::HashMap,
5 sync::{
6 Arc, PoisonError, RwLock,
7 atomic::{AtomicU64, Ordering},
8 },
9};
10
11use redis::{
12 Client, Cmd, Pipeline, RedisError, RedisFuture, RedisResult, ServerError, ServerErrorKind,
13 Value,
14 aio::{ConnectionLike, ConnectionManager, ConnectionManagerConfig, MultiplexedConnection},
15 cluster_async::ClusterConnection,
16 cluster_routing::{MultipleNodeRoutingInfo, RoutingInfo, SingleNodeRoutingInfo},
17 from_redis_value,
18 sentinel::SentinelClient,
19};
20use tokio::sync::Mutex;
21
22/// The settings of every `ConnectionManager` this crate builds. While the server is down, a call
23/// fails at once instead of waiting through redis-rs's default six reconnect attempts, about 9 s,
24/// and the next call connects again. A cache that is down then costs a caller milliseconds, and
25/// `get_or_set` goes straight to its loader. Timeouts stay redis-rs's: 500 ms for a reply, 1 s
26/// for a connection attempt.
27///
28/// Start from it when you build a `ConnectionManager` yourself:
29///
30/// ```no_run
31/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
32/// use std::time::Duration;
33/// use sisa_caching_redis::{RedisCacheStore, connection_config, redis::{Client, aio::ConnectionManager}};
34///
35/// let config = connection_config().set_response_timeout(Some(Duration::from_secs(2)));
36/// let conn = ConnectionManager::new_with_config(Client::open("redis://127.0.0.1:6379")?, config).await?;
37/// let cache = RedisCacheStore::new("app", conn)?;
38/// # Ok(())
39/// # }
40/// ```
41pub fn connection_config() -> ConnectionManagerConfig {
42 ConnectionManagerConfig::new().set_number_of_retries(0)
43}
44
45/// A node's host and port, as the cluster names it.
46pub type NodeAddress = (String, u16);
47
48/// A connection a [`RedisCacheStore`](crate::store::RedisCacheStore) runs on. Keys go wherever the
49/// connection routes them; only `remove_prefix` and `clear` need more, because `SCAN` sees one
50/// node's keys. They scan each node from [`primaries`](Self::primaries).
51pub trait RedisConnection: ConnectionLike + Clone + Send + Sync + 'static {
52 /// The nodes that hold keys. `None` is the one server this connection talks to.
53 fn primaries(&mut self) -> impl Future<Output = RedisResult<Vec<Option<NodeAddress>>>> + Send {
54 async { Ok(vec![None]) }
55 }
56
57 /// Sends `cmd` to `node`, or along the connection when `node` is `None`.
58 fn query_node(
59 &mut self,
60 _node: Option<&NodeAddress>,
61 cmd: &Cmd,
62 ) -> impl Future<Output = RedisResult<Value>> + Send {
63 async move { self.req_packed_command(cmd).await }
64 }
65}
66
67// One server, with or without replicas, so the default methods fit.
68impl RedisConnection for ConnectionManager {}
69
70// One server too, without the reconnects a `ConnectionManager` adds.
71impl RedisConnection for MultiplexedConnection {}
72
73// A cluster: `primaries` asks every primary, and `query_node` sends to one of them.
74impl RedisConnection for ClusterConnection {
75 async fn primaries(&mut self) -> RedisResult<Vec<Option<NodeAddress>>> {
76 // Asked of every primary, so each reply comes keyed by the address this connection reaches
77 // its node at. One reply is enough: a node it lists with a NULL or empty host, as with
78 // `cluster-preferred-endpoint-type unknown-endpoint`, is on the host the command went to.
79 let every_primary = RoutingInfo::MultiNode((MultipleNodeRoutingInfo::AllMasters, None));
80 let replies: HashMap<String, Value> = from_redis_value(
81 self.route_command(redis::cmd("CLUSTER").arg("SLOTS").to_owned(), every_primary)
82 .await?,
83 )?;
84 let Some((asked, ranges)) = replies.into_iter().next() else {
85 return Ok(Vec::new());
86 };
87 let asked = asked
88 .rsplit_once(':')
89 .map_or(asked.as_str(), |(host, _)| host);
90 // One row per slot range: [start, end, primary, replicas...], each node [host, port, ...].
91 let ranges: Vec<Vec<Value>> = from_redis_value(ranges.extract_error()?)?;
92 let mut primaries = Vec::new();
93
94 for range in ranges {
95 let Some(node) = range.into_iter().nth(2) else {
96 continue;
97 };
98 let node: Vec<Value> = from_redis_value(node)?;
99 let mut node = node.into_iter();
100 let host: Option<String> = from_redis_value(node.next().unwrap_or(Value::Nil))?;
101 let address = Some((
102 host.filter(|host| !host.is_empty())
103 .unwrap_or_else(|| asked.to_owned()),
104 from_redis_value(node.next().unwrap_or(Value::Nil))?,
105 ));
106
107 if !primaries.contains(&address) {
108 primaries.push(address);
109 }
110 }
111
112 Ok(primaries)
113 }
114
115 async fn query_node(&mut self, node: Option<&NodeAddress>, cmd: &Cmd) -> RedisResult<Value> {
116 let Some((host, port)) = node else {
117 return self.req_packed_command(cmd).await;
118 };
119 let route = SingleNodeRoutingInfo::ByAddress {
120 host: host.clone(),
121 port: *port,
122 };
123
124 self.route_command(cmd.clone(), RoutingInfo::SingleNode(route))
125 .await
126 }
127}
128
129/// The connection to the current primary, and how many times it has moved, so a clone that saw a
130/// command fail can tell whether another clone already moved.
131#[derive(Clone)]
132struct Primary {
133 generation: u64,
134 client: Client,
135 connection: ConnectionManager,
136}
137
138impl Primary {
139 async fn connect(client: Client, generation: u64) -> RedisResult<Self> {
140 let connection =
141 ConnectionManager::new_with_config(client.clone(), connection_config()).await?;
142
143 Ok(Self {
144 generation,
145 client,
146 connection,
147 })
148 }
149}
150
151/// A replica runs no writes, and a server that is loading runs no commands, so `READONLY` or
152/// `LOADING` means the command didn't run.
153fn refusal(error: &ServerError) -> bool {
154 matches!(
155 error.kind(),
156 Some(ServerErrorKind::ReadOnly | ServerErrorKind::BusyLoading)
157 )
158}
159
160/// Outside a transaction a server error is a reply, not an `Err`.
161fn refused(reply: &Value) -> bool {
162 matches!(reply, Value::ServerError(error) if refusal(error))
163}
164
165/// The command didn't run: the connection was refused, or, inside a transaction, the server
166/// refused one of the aborted commands.
167fn not_run(error: &RedisError) -> bool {
168 error.is_connection_refusal()
169 || error
170 .clone()
171 .into_server_errors()
172 .is_some_and(|errors| errors.iter().any(|(_, aborted)| refusal(aborted)))
173}
174
175/// A connection to the primary that Redis Sentinel names, which follows a failover:
176///
177/// - A command the server refused with `READONLY` or `LOADING`, or that the connection refused to
178/// send, didn't run: the connection asks Sentinel for the primary, moves there, and sends it
179/// once more. A demoted Dragonfly primary answers `LOADING` while it syncs from the new one.
180/// - A command cut off on the way may have run, so it returns its error at once, and the
181/// connection asks Sentinel in the background and moves for the next one.
182/// - While Sentinel names the same primary, the connection stays and reconnects by itself.
183/// - Clones share the connection, so one move moves all of them.
184///
185/// It moves on an error, not on Sentinel's announcement. After a manual failover the old primary
186/// keeps accepting writes until Sentinel demotes it, about ten seconds later, and those writes are
187/// lost. Reads keep answering from it, as a replica, until the next write or a
188/// [`primary_client`](Self::primary_client) call moves the connection.
189///
190/// Its connections use [`connection_config`], so while the primary is down a command fails at
191/// once instead of waiting for the old primary to come back. A command that can't reach the
192/// primary also waits for one Sentinel lookup, which every caller failing at the same time shares,
193/// before it is sent once more.
194///
195/// ```no_run
196/// use sisa_caching_redis::{
197/// RedisCacheStore, SentinelConnection,
198/// redis::sentinel::{SentinelClient, SentinelServerType},
199/// };
200///
201/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
202/// let sentinel = SentinelClient::build(
203/// vec!["redis://10.0.0.1:26379", "redis://10.0.0.2:26379"],
204/// String::from("mymaster"),
205/// None,
206/// SentinelServerType::Master,
207/// )?;
208/// let conn = SentinelConnection::connect(sentinel).await?;
209/// let cache = RedisCacheStore::new("app", conn)?;
210/// # Ok(())
211/// # }
212/// ```
213#[derive(Clone)]
214pub struct SentinelConnection {
215 sentinel: Arc<Mutex<Sentinel>>,
216 /// How many lookups have finished, so a caller that waited for one can tell it did.
217 lookups: Arc<AtomicU64>,
218 primary: Arc<RwLock<Primary>>,
219}
220
221/// The Sentinel client, and why the last lookup failed, if it did, for the callers that waited
222/// for it.
223struct Sentinel {
224 client: SentinelClient,
225 failed: Option<RedisError>,
226}
227
228// Shows how many failovers it has followed; the clients have nothing to show.
229impl std::fmt::Debug for SentinelConnection {
230 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
231 formatter
232 .debug_struct("SentinelConnection")
233 .field("failovers", &self.primary().generation)
234 .finish_non_exhaustive()
235 }
236}
237
238impl SentinelConnection {
239 /// Asks Sentinel for the primary and connects to it. Build `sentinel` with
240 /// `SentinelServerType::Master`.
241 pub async fn connect(mut sentinel: SentinelClient) -> RedisResult<Self> {
242 let primary = Primary::connect(sentinel.async_get_client().await?, 0).await?;
243
244 Ok(Self {
245 sentinel: Arc::new(Mutex::new(Sentinel {
246 client: sentinel,
247 failed: None,
248 })),
249 lookups: Arc::default(),
250 primary: Arc::new(RwLock::new(primary)),
251 })
252 }
253
254 /// Asks Sentinel for the primary, moves this connection there if it isn't there yet, and
255 /// returns a client for it, for connections this one can't carry, such as pub/sub. A
256 /// subscription opened with it and this connection's reads then use the same primary. A
257 /// hybrid store's backplane calls it every `heartbeat_timeout_ms` while subscribed, so reads
258 /// leave a demoted primary even when nothing fails there.
259 pub async fn primary_client(&self) -> RedisResult<Client> {
260 Ok(self.follow(self.primary().generation).await?.client)
261 }
262
263 fn primary(&self) -> Primary {
264 self.primary
265 .read()
266 .unwrap_or_else(PoisonError::into_inner)
267 .clone()
268 }
269
270 /// The lookup [`follow`](Self::follow) runs under the Sentinel lock.
271 async fn look_up(&self, sentinel: &mut SentinelClient, seen: u64) -> RedisResult<Primary> {
272 let current = self.primary();
273
274 if current.generation != seen {
275 return Ok(current);
276 }
277
278 let client = sentinel.async_get_client().await?;
279
280 if client.get_connection_info().addr() == current.client.get_connection_info().addr() {
281 return Ok(current);
282 }
283
284 let primary = Primary::connect(client, current.generation + 1).await?;
285
286 *self.primary.write().unwrap_or_else(PoisonError::into_inner) = primary.clone();
287 Ok(primary)
288 }
289
290 /// Moves to the primary Sentinel names now, unless a clone already moved past `seen` or the
291 /// connection is there already: a connection to a primary that didn't move reconnects by
292 /// itself. One lookup runs at a time, under the Sentinel lock. A caller that finds one running
293 /// waits for it and takes what it found, its error included, as new as its own would be, so
294 /// callers that fail together while the primary is down share one lookup instead of queueing
295 /// for one each. If the caller it waited for gave up before its lookup ended, it looks up
296 /// itself.
297 async fn follow(&self, seen: u64) -> RedisResult<Primary> {
298 let finished = self.lookups.load(Ordering::Relaxed);
299 let mut sentinel = match self.sentinel.try_lock() {
300 Ok(sentinel) => sentinel,
301 Err(_) => {
302 // Lookups are counted under the lock, so this count is the latest.
303 let sentinel = self.sentinel.lock().await;
304
305 if self.lookups.load(Ordering::Relaxed) != finished {
306 return match &sentinel.failed {
307 Some(error) => Err(error.clone()),
308 None => Ok(self.primary()),
309 };
310 }
311
312 sentinel
313 }
314 };
315 let found = self.look_up(&mut sentinel.client, seen).await;
316
317 if let Err(error) = &found {
318 tracing::debug!(%error, "asking Sentinel for the primary failed");
319 }
320 sentinel.failed = found.as_ref().err().cloned();
321 self.lookups.fetch_add(1, Ordering::Relaxed);
322
323 found
324 }
325
326 /// Sends on the current primary. When the reply shows a failover it moves, and sends once
327 /// more if the first try surely didn't run.
328 ///
329 /// `send` is a closure that returns a future, not an async closure: an `AsyncFn` here makes
330 /// the boxed future its callers return need `'static` borrows, and theirs borrow the command.
331 async fn send<T, Fut>(
332 &self,
333 send: impl Fn(ConnectionManager) -> Fut,
334 refused: impl Fn(&T) -> bool,
335 ) -> RedisResult<T>
336 where
337 Fut: Future<Output = RedisResult<T>>,
338 {
339 let Primary {
340 generation,
341 connection,
342 ..
343 } = self.primary();
344 let reply = send(connection).await;
345 let resend = match &reply {
346 Ok(value) if refused(value) => true,
347 Err(error) if not_run(error) => true,
348 // Cut off, so it may have run: the next command goes to the new primary.
349 Err(error) if error.is_io_error() || error.is_connection_dropped() => false,
350 _ => return reply,
351 };
352
353 if resend {
354 send(self.follow(generation).await?.connection).await
355 } else {
356 // The command's own error says more than a failed lookup would, so it returns now,
357 // and the lookup runs on its own task for the commands after it.
358 let follower = self.clone();
359
360 tokio::spawn(async move { _ = follower.follow(generation).await });
361 reply
362 }
363 }
364}
365
366// Sends each command to the current primary, and follows a failover when a reply shows one.
367impl ConnectionLike for SentinelConnection {
368 fn req_packed_command<'a>(&'a mut self, cmd: &'a Cmd) -> RedisFuture<'a, Value> {
369 Box::pin(self.send(
370 move |mut primary| async move { primary.req_packed_command(cmd).await },
371 refused,
372 ))
373 }
374
375 fn req_packed_commands<'a>(
376 &'a mut self,
377 pipe: &'a Pipeline,
378 offset: usize,
379 count: usize,
380 ) -> RedisFuture<'a, Vec<Value>> {
381 Box::pin(self.send(
382 move |mut primary| async move { primary.req_packed_commands(pipe, offset, count).await },
383 |replies: &Vec<Value>| replies.iter().any(refused),
384 ))
385 }
386
387 fn get_db(&self) -> i64 {
388 self.primary().connection.get_db()
389 }
390}
391
392// One primary at a time, so the default methods fit.
393impl RedisConnection for SentinelConnection {}