Skip to main content

sisa_caching_redis/
connection.rs

1//! The connections a Redis store runs on: one server, a cluster, or a primary behind Sentinel.
2
3use std::{
4    collections::HashMap,
5    sync::{
6        Arc, PoisonError, RwLock,
7        atomic::{AtomicU64, Ordering},
8    },
9};
10
11use redis::{
12    Client, Cmd, Pipeline, RedisError, RedisFuture, RedisResult, ServerError, ServerErrorKind,
13    Value,
14    aio::{ConnectionLike, ConnectionManager, ConnectionManagerConfig, MultiplexedConnection},
15    cluster_async::ClusterConnection,
16    cluster_routing::{MultipleNodeRoutingInfo, RoutingInfo, SingleNodeRoutingInfo},
17    from_redis_value,
18    sentinel::SentinelClient,
19};
20use tokio::sync::Mutex;
21
22/// The settings of every `ConnectionManager` this crate builds. While the server is down, a call
23/// fails at once instead of waiting through redis-rs's default six reconnect attempts, about 9 s,
24/// and the next call connects again. A cache that is down then costs a caller milliseconds, and
25/// `get_or_set` goes straight to its loader. Timeouts stay redis-rs's: 500 ms for a reply, 1 s
26/// for a connection attempt.
27///
28/// Start from it when you build a `ConnectionManager` yourself:
29///
30/// ```no_run
31/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
32/// use std::time::Duration;
33/// use sisa_caching_redis::{RedisCacheStore, connection_config, redis::{Client, aio::ConnectionManager}};
34///
35/// let config = connection_config().set_response_timeout(Some(Duration::from_secs(2)));
36/// let conn = ConnectionManager::new_with_config(Client::open("redis://127.0.0.1:6379")?, config).await?;
37/// let cache = RedisCacheStore::new("app", conn)?;
38/// # Ok(())
39/// # }
40/// ```
41pub fn connection_config() -> ConnectionManagerConfig {
42    ConnectionManagerConfig::new().set_number_of_retries(0)
43}
44
45/// A node's host and port, as the cluster names it.
46pub type NodeAddress = (String, u16);
47
48/// A connection a [`RedisCacheStore`](crate::store::RedisCacheStore) runs on. Keys go wherever the
49/// connection routes them; only `remove_prefix` and `clear` need more, because `SCAN` sees one
50/// node's keys. They scan each node from [`primaries`](Self::primaries).
51pub trait RedisConnection: ConnectionLike + Clone + Send + Sync + 'static {
52    /// The nodes that hold keys. `None` is the one server this connection talks to.
53    fn primaries(&mut self) -> impl Future<Output = RedisResult<Vec<Option<NodeAddress>>>> + Send {
54        async { Ok(vec![None]) }
55    }
56
57    /// Sends `cmd` to `node`, or along the connection when `node` is `None`.
58    fn query_node(
59        &mut self,
60        _node: Option<&NodeAddress>,
61        cmd: &Cmd,
62    ) -> impl Future<Output = RedisResult<Value>> + Send {
63        async move { self.req_packed_command(cmd).await }
64    }
65}
66
67// One server, with or without replicas, so the default methods fit.
68impl RedisConnection for ConnectionManager {}
69
70// One server too, without the reconnects a `ConnectionManager` adds.
71impl RedisConnection for MultiplexedConnection {}
72
73// A cluster: `primaries` asks every primary, and `query_node` sends to one of them.
74impl RedisConnection for ClusterConnection {
75    async fn primaries(&mut self) -> RedisResult<Vec<Option<NodeAddress>>> {
76        // Asked of every primary, so each reply comes keyed by the address this connection reaches
77        // its node at. One reply is enough: a node it lists with a NULL or empty host, as with
78        // `cluster-preferred-endpoint-type unknown-endpoint`, is on the host the command went to.
79        let every_primary = RoutingInfo::MultiNode((MultipleNodeRoutingInfo::AllMasters, None));
80        let replies: HashMap<String, Value> = from_redis_value(
81            self.route_command(redis::cmd("CLUSTER").arg("SLOTS").to_owned(), every_primary)
82                .await?,
83        )?;
84        let Some((asked, ranges)) = replies.into_iter().next() else {
85            return Ok(Vec::new());
86        };
87        let asked = asked
88            .rsplit_once(':')
89            .map_or(asked.as_str(), |(host, _)| host);
90        // One row per slot range: [start, end, primary, replicas...], each node [host, port, ...].
91        let ranges: Vec<Vec<Value>> = from_redis_value(ranges.extract_error()?)?;
92        let mut primaries = Vec::new();
93
94        for range in ranges {
95            let Some(node) = range.into_iter().nth(2) else {
96                continue;
97            };
98            let node: Vec<Value> = from_redis_value(node)?;
99            let mut node = node.into_iter();
100            let host: Option<String> = from_redis_value(node.next().unwrap_or(Value::Nil))?;
101            let address = Some((
102                host.filter(|host| !host.is_empty())
103                    .unwrap_or_else(|| asked.to_owned()),
104                from_redis_value(node.next().unwrap_or(Value::Nil))?,
105            ));
106
107            if !primaries.contains(&address) {
108                primaries.push(address);
109            }
110        }
111
112        Ok(primaries)
113    }
114
115    async fn query_node(&mut self, node: Option<&NodeAddress>, cmd: &Cmd) -> RedisResult<Value> {
116        let Some((host, port)) = node else {
117            return self.req_packed_command(cmd).await;
118        };
119        let route = SingleNodeRoutingInfo::ByAddress {
120            host: host.clone(),
121            port: *port,
122        };
123
124        self.route_command(cmd.clone(), RoutingInfo::SingleNode(route))
125            .await
126    }
127}
128
129/// The connection to the current primary, and how many times it has moved, so a clone that saw a
130/// command fail can tell whether another clone already moved.
131#[derive(Clone)]
132struct Primary {
133    generation: u64,
134    client: Client,
135    connection: ConnectionManager,
136}
137
138impl Primary {
139    async fn connect(client: Client, generation: u64) -> RedisResult<Self> {
140        let connection =
141            ConnectionManager::new_with_config(client.clone(), connection_config()).await?;
142
143        Ok(Self {
144            generation,
145            client,
146            connection,
147        })
148    }
149}
150
151/// A replica runs no writes, and a server that is loading runs no commands, so `READONLY` or
152/// `LOADING` means the command didn't run.
153fn refusal(error: &ServerError) -> bool {
154    matches!(
155        error.kind(),
156        Some(ServerErrorKind::ReadOnly | ServerErrorKind::BusyLoading)
157    )
158}
159
160/// Outside a transaction a server error is a reply, not an `Err`.
161fn refused(reply: &Value) -> bool {
162    matches!(reply, Value::ServerError(error) if refusal(error))
163}
164
165/// The command didn't run: the connection was refused, or, inside a transaction, the server
166/// refused one of the aborted commands.
167fn not_run(error: &RedisError) -> bool {
168    error.is_connection_refusal()
169        || error
170            .clone()
171            .into_server_errors()
172            .is_some_and(|errors| errors.iter().any(|(_, aborted)| refusal(aborted)))
173}
174
175/// A connection to the primary that Redis Sentinel names, which follows a failover:
176///
177/// - A command the server refused with `READONLY` or `LOADING`, or that the connection refused to
178///   send, didn't run: the connection asks Sentinel for the primary, moves there, and sends it
179///   once more. A demoted Dragonfly primary answers `LOADING` while it syncs from the new one.
180/// - A command cut off on the way may have run, so it returns its error at once, and the
181///   connection asks Sentinel in the background and moves for the next one.
182/// - While Sentinel names the same primary, the connection stays and reconnects by itself.
183/// - Clones share the connection, so one move moves all of them.
184///
185/// It moves on an error, not on Sentinel's announcement. After a manual failover the old primary
186/// keeps accepting writes until Sentinel demotes it, about ten seconds later, and those writes are
187/// lost. Reads keep answering from it, as a replica, until the next write or a
188/// [`primary_client`](Self::primary_client) call moves the connection.
189///
190/// Its connections use [`connection_config`], so while the primary is down a command fails at
191/// once instead of waiting for the old primary to come back. A command that can't reach the
192/// primary also waits for one Sentinel lookup, which every caller failing at the same time shares,
193/// before it is sent once more.
194///
195/// ```no_run
196/// use sisa_caching_redis::{
197///     RedisCacheStore, SentinelConnection,
198///     redis::sentinel::{SentinelClient, SentinelServerType},
199/// };
200///
201/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
202/// let sentinel = SentinelClient::build(
203///     vec!["redis://10.0.0.1:26379", "redis://10.0.0.2:26379"],
204///     String::from("mymaster"),
205///     None,
206///     SentinelServerType::Master,
207/// )?;
208/// let conn = SentinelConnection::connect(sentinel).await?;
209/// let cache = RedisCacheStore::new("app", conn)?;
210/// # Ok(())
211/// # }
212/// ```
213#[derive(Clone)]
214pub struct SentinelConnection {
215    sentinel: Arc<Mutex<Sentinel>>,
216    /// How many lookups have finished, so a caller that waited for one can tell it did.
217    lookups: Arc<AtomicU64>,
218    primary: Arc<RwLock<Primary>>,
219}
220
221/// The Sentinel client, and why the last lookup failed, if it did, for the callers that waited
222/// for it.
223struct Sentinel {
224    client: SentinelClient,
225    failed: Option<RedisError>,
226}
227
228// Shows how many failovers it has followed; the clients have nothing to show.
229impl std::fmt::Debug for SentinelConnection {
230    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
231        formatter
232            .debug_struct("SentinelConnection")
233            .field("failovers", &self.primary().generation)
234            .finish_non_exhaustive()
235    }
236}
237
238impl SentinelConnection {
239    /// Asks Sentinel for the primary and connects to it. Build `sentinel` with
240    /// `SentinelServerType::Master`.
241    pub async fn connect(mut sentinel: SentinelClient) -> RedisResult<Self> {
242        let primary = Primary::connect(sentinel.async_get_client().await?, 0).await?;
243
244        Ok(Self {
245            sentinel: Arc::new(Mutex::new(Sentinel {
246                client: sentinel,
247                failed: None,
248            })),
249            lookups: Arc::default(),
250            primary: Arc::new(RwLock::new(primary)),
251        })
252    }
253
254    /// Asks Sentinel for the primary, moves this connection there if it isn't there yet, and
255    /// returns a client for it, for connections this one can't carry, such as pub/sub. A
256    /// subscription opened with it and this connection's reads then use the same primary. A
257    /// hybrid store's backplane calls it every `heartbeat_timeout_ms` while subscribed, so reads
258    /// leave a demoted primary even when nothing fails there.
259    pub async fn primary_client(&self) -> RedisResult<Client> {
260        Ok(self.follow(self.primary().generation).await?.client)
261    }
262
263    fn primary(&self) -> Primary {
264        self.primary
265            .read()
266            .unwrap_or_else(PoisonError::into_inner)
267            .clone()
268    }
269
270    /// The lookup [`follow`](Self::follow) runs under the Sentinel lock.
271    async fn look_up(&self, sentinel: &mut SentinelClient, seen: u64) -> RedisResult<Primary> {
272        let current = self.primary();
273
274        if current.generation != seen {
275            return Ok(current);
276        }
277
278        let client = sentinel.async_get_client().await?;
279
280        if client.get_connection_info().addr() == current.client.get_connection_info().addr() {
281            return Ok(current);
282        }
283
284        let primary = Primary::connect(client, current.generation + 1).await?;
285
286        *self.primary.write().unwrap_or_else(PoisonError::into_inner) = primary.clone();
287        Ok(primary)
288    }
289
290    /// Moves to the primary Sentinel names now, unless a clone already moved past `seen` or the
291    /// connection is there already: a connection to a primary that didn't move reconnects by
292    /// itself. One lookup runs at a time, under the Sentinel lock. A caller that finds one running
293    /// waits for it and takes what it found, its error included, as new as its own would be, so
294    /// callers that fail together while the primary is down share one lookup instead of queueing
295    /// for one each. If the caller it waited for gave up before its lookup ended, it looks up
296    /// itself.
297    async fn follow(&self, seen: u64) -> RedisResult<Primary> {
298        let finished = self.lookups.load(Ordering::Relaxed);
299        let mut sentinel = match self.sentinel.try_lock() {
300            Ok(sentinel) => sentinel,
301            Err(_) => {
302                // Lookups are counted under the lock, so this count is the latest.
303                let sentinel = self.sentinel.lock().await;
304
305                if self.lookups.load(Ordering::Relaxed) != finished {
306                    return match &sentinel.failed {
307                        Some(error) => Err(error.clone()),
308                        None => Ok(self.primary()),
309                    };
310                }
311
312                sentinel
313            }
314        };
315        let found = self.look_up(&mut sentinel.client, seen).await;
316
317        if let Err(error) = &found {
318            tracing::debug!(%error, "asking Sentinel for the primary failed");
319        }
320        sentinel.failed = found.as_ref().err().cloned();
321        self.lookups.fetch_add(1, Ordering::Relaxed);
322
323        found
324    }
325
326    /// Sends on the current primary. When the reply shows a failover it moves, and sends once
327    /// more if the first try surely didn't run.
328    ///
329    /// `send` is a closure that returns a future, not an async closure: an `AsyncFn` here makes
330    /// the boxed future its callers return need `'static` borrows, and theirs borrow the command.
331    async fn send<T, Fut>(
332        &self,
333        send: impl Fn(ConnectionManager) -> Fut,
334        refused: impl Fn(&T) -> bool,
335    ) -> RedisResult<T>
336    where
337        Fut: Future<Output = RedisResult<T>>,
338    {
339        let Primary {
340            generation,
341            connection,
342            ..
343        } = self.primary();
344        let reply = send(connection).await;
345        let resend = match &reply {
346            Ok(value) if refused(value) => true,
347            Err(error) if not_run(error) => true,
348            // Cut off, so it may have run: the next command goes to the new primary.
349            Err(error) if error.is_io_error() || error.is_connection_dropped() => false,
350            _ => return reply,
351        };
352
353        if resend {
354            send(self.follow(generation).await?.connection).await
355        } else {
356            // The command's own error says more than a failed lookup would, so it returns now,
357            // and the lookup runs on its own task for the commands after it.
358            let follower = self.clone();
359
360            tokio::spawn(async move { _ = follower.follow(generation).await });
361            reply
362        }
363    }
364}
365
366// Sends each command to the current primary, and follows a failover when a reply shows one.
367impl ConnectionLike for SentinelConnection {
368    fn req_packed_command<'a>(&'a mut self, cmd: &'a Cmd) -> RedisFuture<'a, Value> {
369        Box::pin(self.send(
370            move |mut primary| async move { primary.req_packed_command(cmd).await },
371            refused,
372        ))
373    }
374
375    fn req_packed_commands<'a>(
376        &'a mut self,
377        pipe: &'a Pipeline,
378        offset: usize,
379        count: usize,
380    ) -> RedisFuture<'a, Vec<Value>> {
381        Box::pin(self.send(
382            move |mut primary| async move { primary.req_packed_commands(pipe, offset, count).await },
383            |replies: &Vec<Value>| replies.iter().any(refused),
384        ))
385    }
386
387    fn get_db(&self) -> i64 {
388        self.primary().connection.get_db()
389    }
390}
391
392// One primary at a time, so the default methods fit.
393impl RedisConnection for SentinelConnection {}