Skip to main content

cortexkit_bus_client/
buckets.rs

1use std::{future::Future, time::Duration};
2
3use async_nats::jetstream::{
4    self,
5    consumer::{push, DeliverPolicy},
6    kv,
7    stream::ConsumerErrorKind,
8};
9use bytes::Bytes;
10use cortexkit_bus_naming::{ModuleBucket, MODULE_BUCKETS};
11use cortexkit_bus_nats::NatsConnection;
12use futures_util::StreamExt;
13
14use crate::{binding::ConnectionBinding, BusClient, Error, Result};
15
16/// A current value and its bucket-wide JetStream revision.
17#[derive(Debug, Clone, PartialEq, Eq)]
18#[non_exhaustive]
19pub struct BucketEntry {
20    pub value: Bytes,
21    pub revision: u64,
22}
23
24/// A snapshot/live change, or the boundary after the initial snapshot.
25#[derive(Debug, Clone, PartialEq, Eq)]
26#[non_exhaustive]
27pub enum BucketItem {
28    Put {
29        key: String,
30        revision: u64,
31        value: Bytes,
32    },
33    Delete {
34        key: String,
35        revision: u64,
36    },
37    CaughtUp,
38}
39
40/// A declared module bucket. Keep its BusClient alive for credential upkeep.
41#[derive(Clone)]
42pub struct Bucket {
43    binding: ConnectionBinding,
44    name: String,
45    declaration: &'static ModuleBucket,
46    module: String,
47}
48
49impl Bucket {
50    pub(crate) fn attach(client: &BusClient, owner: &str, bucket: &str) -> Result<Self> {
51        let declaration = MODULE_BUCKETS
52            .iter()
53            .find(|entry| entry.owner == owner && entry.name == bucket)
54            .ok_or_else(|| Error::BucketNotDeclared {
55                owner: owner.into(),
56                bucket: bucket.into(),
57            })?;
58        let name = client.names.module_bucket_name(owner, bucket)?;
59        if client.module != declaration.owner
60            && !declaration.readers.contains(&client.module.as_str())
61        {
62            return Err(Error::BucketNotReadable {
63                module: client.module.clone(),
64                bucket: name,
65            });
66        }
67        Ok(Self {
68            binding: ConnectionBinding::new(client),
69            name,
70            declaration,
71            module: client.module.clone(),
72        })
73    }
74
75    fn require_owner(&self) -> Result<()> {
76        if self.module != self.declaration.owner {
77            return Err(Error::BucketNotOwned {
78                module: self.module.clone(),
79                bucket: self.name.clone(),
80            });
81        }
82        Ok(())
83    }
84
85    async fn store(&self) -> Result<kv::Store> {
86        jetstream::new(self.binding.snapshot().client())
87            .get_key_value(&self.name)
88            .await
89            .map_err(|error| Error::Nats(error.to_string()))
90    }
91
92    /// Gets the latest value; absent and deleted keys both return None.
93    pub async fn get(&self, key: &str) -> Result<Option<BucketEntry>> {
94        Ok(self
95            .store()
96            .await?
97            .entry(key)
98            .await
99            .map_err(|error| Error::Nats(error.to_string()))?
100            .filter(|entry| entry.operation == kv::Operation::Put)
101            .map(|entry| BucketEntry {
102                value: entry.value,
103                revision: entry.revision,
104            }))
105    }
106
107    /// Writes as the declared owner, refusing oversized values before any request.
108    pub async fn put(&self, key: &str, value: impl Into<Bytes>) -> Result<u64> {
109        self.require_owner()?;
110        let value = value.into();
111        if value.len() > self.declaration.max_value_bytes as usize {
112            return Err(Error::BucketValueTooLarge {
113                size: value.len(),
114                cap: self.declaration.max_value_bytes,
115            });
116        }
117        self.store()
118            .await?
119            .put(key, value)
120            .await
121            .map_err(|error| Error::Nats(error.to_string()))
122    }
123
124    /// Stores a retained delete marker and returns its publish acknowledgement's revision.
125    pub async fn delete(&self, key: &str) -> Result<u64> {
126        self.require_owner()?;
127        validate_key(key)?;
128        let mut headers = async_nats::HeaderMap::new();
129        headers.insert("KV-Operation", "DEL");
130        // Store::delete discards the acknowledgement; publishing the same marker
131        // directly preserves its revision without a racing read-back.
132        let ack = jetstream::new(self.binding.snapshot().client())
133            .publish_with_headers(format!("$KV.{}.{key}", self.name), headers, Bytes::new())
134            .await
135            .map_err(|error| Error::Nats(error.to_string()))?
136            .await
137            .map_err(|error| Error::Nats(error.to_string()))?;
138        Ok(ack.sequence)
139    }
140
141    /// Delivers current entries, one CaughtUp marker (even when empty), then changes.
142    /// Uses only unnamed ephemeral consumers; dropped watches release their slots
143    /// after the server's 30-second inactivity threshold, not immediately.
144    /// New watches fail immediately with WatchCapacity when all slots are full.
145    /// An existing watch rebinding after credential reissue logs once and retries
146    /// capacity with 1-to-5-second backoff for at most 60 seconds, then returns
147    /// WatchCapacity if no slot became available. Terminal maintenance interrupts
148    /// that wait with the same transport error as EventReader.
149    ///
150    /// After reconnecting, delivers each key whose latest retained revision is
151    /// newer than the last delivered revision, without repeating a revision.
152    /// Intermediate values overwritten while disconnected are not delivered;
153    /// readers needing every change must use an event stream, not a bucket.
154    /// Delete markers remain the key's latest entry and survive reconnects.
155    /// Owners must use delete, never purge: removing a marker hides the deletion.
156    /// Keep the BusClient alive; terminal renewal ends next() with Error::Transport,
157    /// just as EventReader does.
158    pub async fn watch(&self) -> Result<BucketWatch> {
159        let mut watch = BucketWatch {
160            bucket: self.clone(),
161            subscription: None,
162            revision: 0,
163            caught_up: false,
164            marker_pending: false,
165            opened: false,
166        };
167        watch.bind().await?;
168        Ok(watch)
169    }
170}
171
172// Match async-nats KV's key grammar when publishing delete markers directly.
173fn validate_key(key: &str) -> Result<()> {
174    if key.is_empty()
175        || key.starts_with('.')
176        || key.ends_with('.')
177        || !key
178            .bytes()
179            .all(|byte| byte.is_ascii_alphanumeric() || b"-/_=.".contains(&byte))
180    {
181        return Err(Error::Nats("invalid KV key".into()));
182    }
183    Ok(())
184}
185
186/// A revision-resuming snapshot and live watch. Cancellation of next() is safe.
187pub struct BucketWatch {
188    bucket: Bucket,
189    subscription: Option<push::Ordered>,
190    revision: u64,
191    caught_up: bool,
192    marker_pending: bool,
193    opened: bool,
194}
195
196impl BucketWatch {
197    async fn bind(&mut self) -> Result<()> {
198        // Drop the old inbox before opening another consumer. Module grants do
199        // not include consumer deletion; the server reaps inactive consumers.
200        self.subscription = None;
201        let name = &self.bucket.name;
202        let revision = self.revision;
203        let (subscription, empty) = bind_watch(
204            &mut self.bucket.binding,
205            name,
206            self.opened,
207            |connection| async move {
208                let context = jetstream::new(connection.client());
209                let consumer = context
210                    .create_consumer_on_stream(
211                        push::OrderedConfig {
212                            deliver_subject: connection.client().new_inbox(),
213                            filter_subject: format!("$KV.{name}.>"),
214                            deliver_policy: if revision == 0 {
215                                DeliverPolicy::LastPerSubject
216                            } else {
217                                DeliverPolicy::ByStartSequence {
218                                    start_sequence: revision + 1,
219                                }
220                            },
221                            ..Default::default()
222                        },
223                        format!("KV_{name}"),
224                    )
225                    .await
226                    .map_err(|error| consumer_error(name, error))?;
227                let empty = consumer.cached_info().num_pending == 0;
228                Ok((
229                    consumer
230                        .messages()
231                        .await
232                        .map_err(|error| Error::Nats(error.to_string()))?,
233                    empty,
234                ))
235            },
236        )
237        .await?;
238        self.subscription = Some(subscription);
239        self.opened = true;
240        self.marker_pending = !self.caught_up && empty;
241        Ok(())
242    }
243
244    /// Returns one change or the initial snapshot boundary, never acknowledging writes.
245    pub async fn next(&mut self) -> Result<BucketItem> {
246        loop {
247            if self.bucket.binding.has_changed() || self.subscription.is_none() {
248                self.bind().await?;
249            }
250            tokio::select! {
251                biased;
252                changed = self.bucket.binding.changed() => { changed?; self.subscription = None; }
253                () = std::future::ready(()), if self.marker_pending => {
254                    self.marker_pending = false;
255                    self.caught_up = true;
256                    return Ok(BucketItem::CaughtUp);
257                }
258                message = self.subscription.as_mut().expect("bound watch").next() => {
259                    let message = message.ok_or_else(|| Error::Nats("bucket watch closed".into()))?
260                        .map_err(|error| Error::Nats(error.to_string()))?;
261                    let info = message.info().map_err(|error| Error::Nats(error.to_string()))?;
262                    if !self.caught_up && info.pending == 0 { self.marker_pending = true; }
263                    if !advance_revision(&mut self.revision, info.stream_sequence) { continue; }
264                    let key = message.subject.strip_prefix(&format!("$KV.{}.", self.bucket.name))
265                        .ok_or(Error::Reply("bucket watch subject mismatch"))?.to_owned();
266                    let revision = info.stream_sequence;
267                    let deleted = message.headers.as_ref().and_then(|headers| headers.get("KV-Operation"))
268                        .is_some_and(|operation| matches!(operation.as_str(), "DEL" | "PURGE"));
269                    return Ok(if deleted { BucketItem::Delete { key, revision } } else {
270                        BucketItem::Put { key, revision, value: message.payload.clone() }
271                    });
272                }
273            }
274        }
275    }
276}
277
278pub(crate) async fn bind_watch<T, F, Fut>(
279    binding: &mut ConnectionBinding,
280    bucket: &str,
281    retry_capacity: bool,
282    mut open: F,
283) -> Result<T>
284where
285    F: FnMut(NatsConnection) -> Fut,
286    Fut: Future<Output = Result<T>>,
287{
288    let mut deadline = None;
289    let mut delay = Duration::from_secs(1);
290    loop {
291        let attempt = binding.bind(&mut open);
292        let result = if let Some(deadline) = deadline {
293            tokio::time::timeout_at(deadline, attempt)
294                .await
295                .unwrap_or_else(|_| {
296                    Err(Error::WatchCapacity {
297                        bucket: bucket.into(),
298                    })
299                })
300        } else {
301            attempt.await
302        };
303        match result {
304            Err(Error::WatchCapacity { .. }) if retry_capacity => {
305                let deadline = *deadline.get_or_insert_with(|| {
306                    tracing::warn!(
307                        bucket,
308                        "waiting for bucket watch consumer capacity during rebind"
309                    );
310                    tokio::time::Instant::now() + Duration::from_secs(60)
311                });
312                let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
313                if remaining.is_zero() {
314                    return Err(Error::WatchCapacity {
315                        bucket: bucket.into(),
316                    });
317                }
318                tokio::select! {
319                    biased;
320                    changed = binding.changed() => { changed?; }
321                    () = tokio::time::sleep(delay.min(remaining)) => {}
322                }
323                delay = (delay * 2).min(Duration::from_secs(5));
324            }
325            result => return result,
326        }
327    }
328}
329
330fn advance_revision(last: &mut u64, revision: u64) -> bool {
331    if revision <= *last {
332        return false;
333    }
334    *last = revision;
335    true
336}
337
338fn consumer_error(bucket: &str, error: jetstream::stream::ConsumerError) -> Error {
339    if matches!(error.kind(), ConsumerErrorKind::JetStream(ref server)
340        if server.error_code() == jetstream::ErrorCode::MAXIMUM_CONSUMERS_LIMIT)
341    {
342        Error::WatchCapacity {
343            bucket: bucket.into(),
344        }
345    } else {
346        Error::Nats(error.to_string())
347    }
348}
349
350#[cfg(test)]
351mod tests {
352    #[test]
353    fn bucket_revision_deduplication() {
354        let mut last = 12;
355        assert!(!super::advance_revision(&mut last, 12));
356        assert!(!super::advance_revision(&mut last, 11));
357        assert_eq!(last, 12);
358        assert!(super::advance_revision(&mut last, 15));
359        assert_eq!(last, 15);
360    }
361}