1use std::{future::Future, time::Duration};
2
3use async_nats::jetstream::{
4 self,
5 consumer::{push, DeliverPolicy},
6 kv,
7 stream::ConsumerErrorKind,
8};
9use bytes::Bytes;
10use cortexkit_bus_naming::{ModuleBucket, MODULE_BUCKETS};
11use cortexkit_bus_nats::NatsConnection;
12use futures_util::StreamExt;
13
14use crate::{binding::ConnectionBinding, BusClient, Error, Result};
15
16#[derive(Debug, Clone, PartialEq, Eq)]
18#[non_exhaustive]
19pub struct BucketEntry {
20 pub value: Bytes,
21 pub revision: u64,
22}
23
24#[derive(Debug, Clone, PartialEq, Eq)]
26#[non_exhaustive]
27pub enum BucketItem {
28 Put {
29 key: String,
30 revision: u64,
31 value: Bytes,
32 },
33 Delete {
34 key: String,
35 revision: u64,
36 },
37 CaughtUp,
38}
39
40#[derive(Clone)]
42pub struct Bucket {
43 binding: ConnectionBinding,
44 name: String,
45 declaration: &'static ModuleBucket,
46 module: String,
47}
48
49impl Bucket {
50 pub(crate) fn attach(client: &BusClient, owner: &str, bucket: &str) -> Result<Self> {
51 let declaration = MODULE_BUCKETS
52 .iter()
53 .find(|entry| entry.owner == owner && entry.name == bucket)
54 .ok_or_else(|| Error::BucketNotDeclared {
55 owner: owner.into(),
56 bucket: bucket.into(),
57 })?;
58 let name = client.names.module_bucket_name(owner, bucket)?;
59 if client.module != declaration.owner
60 && !declaration.readers.contains(&client.module.as_str())
61 {
62 return Err(Error::BucketNotReadable {
63 module: client.module.clone(),
64 bucket: name,
65 });
66 }
67 Ok(Self {
68 binding: ConnectionBinding::new(client),
69 name,
70 declaration,
71 module: client.module.clone(),
72 })
73 }
74
75 fn require_owner(&self) -> Result<()> {
76 if self.module != self.declaration.owner {
77 return Err(Error::BucketNotOwned {
78 module: self.module.clone(),
79 bucket: self.name.clone(),
80 });
81 }
82 Ok(())
83 }
84
85 async fn store(&self) -> Result<kv::Store> {
86 jetstream::new(self.binding.snapshot().client())
87 .get_key_value(&self.name)
88 .await
89 .map_err(|error| Error::Nats(error.to_string()))
90 }
91
92 pub async fn get(&self, key: &str) -> Result<Option<BucketEntry>> {
94 Ok(self
95 .store()
96 .await?
97 .entry(key)
98 .await
99 .map_err(|error| Error::Nats(error.to_string()))?
100 .filter(|entry| entry.operation == kv::Operation::Put)
101 .map(|entry| BucketEntry {
102 value: entry.value,
103 revision: entry.revision,
104 }))
105 }
106
107 pub async fn put(&self, key: &str, value: impl Into<Bytes>) -> Result<u64> {
109 self.require_owner()?;
110 let value = value.into();
111 if value.len() > self.declaration.max_value_bytes as usize {
112 return Err(Error::BucketValueTooLarge {
113 size: value.len(),
114 cap: self.declaration.max_value_bytes,
115 });
116 }
117 self.store()
118 .await?
119 .put(key, value)
120 .await
121 .map_err(|error| Error::Nats(error.to_string()))
122 }
123
124 pub async fn delete(&self, key: &str) -> Result<u64> {
126 self.require_owner()?;
127 validate_key(key)?;
128 let mut headers = async_nats::HeaderMap::new();
129 headers.insert("KV-Operation", "DEL");
130 let ack = jetstream::new(self.binding.snapshot().client())
133 .publish_with_headers(format!("$KV.{}.{key}", self.name), headers, Bytes::new())
134 .await
135 .map_err(|error| Error::Nats(error.to_string()))?
136 .await
137 .map_err(|error| Error::Nats(error.to_string()))?;
138 Ok(ack.sequence)
139 }
140
141 pub async fn watch(&self) -> Result<BucketWatch> {
159 let mut watch = BucketWatch {
160 bucket: self.clone(),
161 subscription: None,
162 revision: 0,
163 caught_up: false,
164 marker_pending: false,
165 opened: false,
166 };
167 watch.bind().await?;
168 Ok(watch)
169 }
170}
171
172fn validate_key(key: &str) -> Result<()> {
174 if key.is_empty()
175 || key.starts_with('.')
176 || key.ends_with('.')
177 || !key
178 .bytes()
179 .all(|byte| byte.is_ascii_alphanumeric() || b"-/_=.".contains(&byte))
180 {
181 return Err(Error::Nats("invalid KV key".into()));
182 }
183 Ok(())
184}
185
186pub struct BucketWatch {
188 bucket: Bucket,
189 subscription: Option<push::Ordered>,
190 revision: u64,
191 caught_up: bool,
192 marker_pending: bool,
193 opened: bool,
194}
195
196impl BucketWatch {
197 async fn bind(&mut self) -> Result<()> {
198 self.subscription = None;
201 let name = &self.bucket.name;
202 let revision = self.revision;
203 let (subscription, empty) = bind_watch(
204 &mut self.bucket.binding,
205 name,
206 self.opened,
207 |connection| async move {
208 let context = jetstream::new(connection.client());
209 let consumer = context
210 .create_consumer_on_stream(
211 push::OrderedConfig {
212 deliver_subject: connection.client().new_inbox(),
213 filter_subject: format!("$KV.{name}.>"),
214 deliver_policy: if revision == 0 {
215 DeliverPolicy::LastPerSubject
216 } else {
217 DeliverPolicy::ByStartSequence {
218 start_sequence: revision + 1,
219 }
220 },
221 ..Default::default()
222 },
223 format!("KV_{name}"),
224 )
225 .await
226 .map_err(|error| consumer_error(name, error))?;
227 let empty = consumer.cached_info().num_pending == 0;
228 Ok((
229 consumer
230 .messages()
231 .await
232 .map_err(|error| Error::Nats(error.to_string()))?,
233 empty,
234 ))
235 },
236 )
237 .await?;
238 self.subscription = Some(subscription);
239 self.opened = true;
240 self.marker_pending = !self.caught_up && empty;
241 Ok(())
242 }
243
244 pub async fn next(&mut self) -> Result<BucketItem> {
246 loop {
247 if self.bucket.binding.has_changed() || self.subscription.is_none() {
248 self.bind().await?;
249 }
250 tokio::select! {
251 biased;
252 changed = self.bucket.binding.changed() => { changed?; self.subscription = None; }
253 () = std::future::ready(()), if self.marker_pending => {
254 self.marker_pending = false;
255 self.caught_up = true;
256 return Ok(BucketItem::CaughtUp);
257 }
258 message = self.subscription.as_mut().expect("bound watch").next() => {
259 let message = match message
260 .ok_or_else(|| Error::Nats("bucket watch closed".into()))
261 .and_then(|message| message.map_err(|error| Error::Nats(error.to_string())))
262 {
263 Ok(message) => message,
264 Err(error) => {
265 self.bucket.binding.recover(error).await?;
268 self.subscription = None;
269 continue;
270 }
271 };
272 let info = message.info().map_err(|error| Error::Nats(error.to_string()))?;
273 if !self.caught_up && info.pending == 0 { self.marker_pending = true; }
274 if !advance_revision(&mut self.revision, info.stream_sequence) { continue; }
275 let key = message.subject.strip_prefix(&format!("$KV.{}.", self.bucket.name))
276 .ok_or(Error::Reply("bucket watch subject mismatch"))?.to_owned();
277 let revision = info.stream_sequence;
278 let deleted = message.headers.as_ref().and_then(|headers| headers.get("KV-Operation"))
279 .is_some_and(|operation| matches!(operation.as_str(), "DEL" | "PURGE"));
280 return Ok(if deleted { BucketItem::Delete { key, revision } } else {
281 BucketItem::Put { key, revision, value: message.payload.clone() }
282 });
283 }
284 }
285 }
286 }
287}
288
289pub(crate) async fn bind_watch<T, F, Fut>(
290 binding: &mut ConnectionBinding,
291 bucket: &str,
292 retry_capacity: bool,
293 mut open: F,
294) -> Result<T>
295where
296 F: FnMut(NatsConnection) -> Fut,
297 Fut: Future<Output = Result<T>>,
298{
299 let mut deadline = None;
300 let mut delay = Duration::from_secs(1);
301 loop {
302 let attempt = binding.bind(&mut open);
303 let result = if let Some(deadline) = deadline {
304 tokio::time::timeout_at(deadline, attempt)
305 .await
306 .unwrap_or_else(|_| {
307 Err(Error::WatchCapacity {
308 bucket: bucket.into(),
309 })
310 })
311 } else {
312 attempt.await
313 };
314 match result {
315 Err(Error::WatchCapacity { .. }) if retry_capacity => {
316 let deadline = *deadline.get_or_insert_with(|| {
317 #[cfg(test)]
318 {
319 binding.bucket_watch_capacity_warnings += 1;
320 }
321 tracing::warn!(
322 bucket,
323 "waiting for bucket watch consumer capacity during rebind"
324 );
325 tokio::time::Instant::now() + Duration::from_secs(60)
326 });
327 let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
328 if remaining.is_zero() {
329 return Err(Error::WatchCapacity {
330 bucket: bucket.into(),
331 });
332 }
333 tokio::select! {
334 biased;
335 changed = binding.changed() => { changed?; }
336 () = tokio::time::sleep(delay.min(remaining)) => {}
337 }
338 delay = (delay * 2).min(Duration::from_secs(5));
339 }
340 result => return result,
341 }
342 }
343}
344
345fn advance_revision(last: &mut u64, revision: u64) -> bool {
346 if revision <= *last {
347 return false;
348 }
349 *last = revision;
350 true
351}
352
353fn consumer_error(bucket: &str, error: jetstream::stream::ConsumerError) -> Error {
354 if matches!(error.kind(), ConsumerErrorKind::JetStream(ref server)
355 if server.error_code() == jetstream::ErrorCode::MAXIMUM_CONSUMERS_LIMIT)
356 {
357 Error::WatchCapacity {
358 bucket: bucket.into(),
359 }
360 } else {
361 Error::Nats(error.to_string())
362 }
363}
364
365#[cfg(test)]
366mod tests {
367 #[test]
368 fn bucket_revision_deduplication() {
369 let mut last = 12;
370 assert!(!super::advance_revision(&mut last, 12));
371 assert!(!super::advance_revision(&mut last, 11));
372 assert_eq!(last, 12);
373 assert!(super::advance_revision(&mut last, 15));
374 assert_eq!(last, 15);
375 }
376}