Skip to main content

cortexkit_bus_client/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod binding;
4mod buckets;
5mod credentials;
6mod events;
7
8pub use buckets::{Bucket, BucketEntry, BucketItem, BucketWatch};
9pub use events::{EventDelivery, EventItem, EventReader, MalformedDelivery, MalformedReason};
10
11use std::{
12    sync::{Arc, RwLock},
13    time::Duration,
14};
15
16use bytes::Bytes;
17use cortexkit_bus_naming::{validate_token, AccountNames, TokenKind};
18use cortexkit_bus_nats::{ConnectConfig, NatsConnection};
19use cortexkit_bus_trait::{ContentDigest, Headers, PublishReceipt, Stream};
20use subc_client_rs::SubcConsumer;
21use tokio::sync::watch;
22
23use credentials::{CredentialManager, SubcTransport};
24
25/// Errors contain refusal codes, not credential JWTs or nonce signatures.
26#[derive(Debug, thiserror::Error)]
27#[non_exhaustive]
28pub enum Error {
29    #[error("ck-bus refused the request: {0}")]
30    Refused(String),
31    #[error("ck-bus request failed: {0}")]
32    Transport(String),
33    #[error("invalid ck-bus reply: {0}")]
34    Reply(&'static str),
35    #[error(transparent)]
36    Naming(#[from] cortexkit_bus_naming::NamingError),
37    #[error(transparent)]
38    Bus(#[from] cortexkit_bus_trait::BusError),
39    #[error("NATS operation failed: {0}")]
40    Nats(String),
41    #[error("event body is {size} bytes; the maximum is 4096 bytes")]
42    EventBodyTooLarge { size: usize },
43    #[error("durable {durable} is not owned by module {module}")]
44    DurableNotOwned { durable: String, module: String },
45    #[error("module bucket {owner}.{bucket} is not declared")]
46    BucketNotDeclared { owner: String, bucket: String },
47    #[error("module {module} may not read bucket {bucket}")]
48    BucketNotReadable { module: String, bucket: String },
49    #[error("module {module} does not own bucket {bucket}")]
50    BucketNotOwned { module: String, bucket: String },
51    #[error("bucket value is {size} bytes; the maximum is {cap} bytes")]
52    BucketValueTooLarge { size: usize, cap: u32 },
53    #[error("bucket {bucket} has no watch consumer slots available")]
54    WatchCapacity { bucket: String },
55}
56
57pub type Result<T> = std::result::Result<T, Error>;
58
59/// Options for one module's bus connection.
60#[derive(Clone, Debug)]
61#[non_exhaustive]
62pub struct BusClientOptions {
63    module: String,
64    pub request_timeout: Duration,
65}
66
67impl BusClientOptions {
68    pub fn new(module: impl Into<String>) -> Result<Self> {
69        let module = module.into();
70        validate_token(TokenKind::ModuleId, &module)?;
71        Ok(Self {
72            module,
73            request_timeout: Duration::from_secs(5),
74        })
75    }
76}
77
78/// A module's current connection and the task that maintains its credentials.
79pub struct BusClient {
80    connection: Arc<RwLock<NatsConnection>>,
81    names: AccountNames,
82    module: String,
83    changes: watch::Receiver<u64>,
84    task: tokio::task::JoinHandle<()>,
85}
86
87impl Drop for BusClient {
88    fn drop(&mut self) {
89        self.task.abort();
90    }
91}
92
93impl BusClient {
94    /// Issues a credential for the module identity that the subc daemon
95    /// authenticates from the module's launch nonce.
96    pub async fn connect(subc: Arc<SubcConsumer>, opts: BusClientOptions) -> Result<Self> {
97        let transport = SubcTransport::new(subc, &opts.module, opts.request_timeout);
98        // One module process can keep several connection snapshots, but issuing
99        // independently for them would revoke the other snapshots' credentials.
100        static MANAGERS: std::sync::LazyLock<
101            tokio::sync::Mutex<
102                std::collections::HashMap<String, std::sync::Weak<CredentialManager>>,
103            >,
104        > = std::sync::LazyLock::new(Default::default);
105        let mut managers = MANAGERS.lock().await;
106        let manager = match managers
107            .get(&opts.module)
108            .and_then(std::sync::Weak::upgrade)
109        {
110            Some(manager) => manager,
111            None => {
112                let manager = CredentialManager::start(Arc::new(transport)).await?;
113                managers.insert(opts.module.clone(), Arc::downgrade(&manager));
114                manager
115            }
116        };
117        drop(managers);
118        Self::connect_manager(manager, opts).await
119    }
120
121    async fn connect_manager(
122        manager: Arc<CredentialManager>,
123        opts: BusClientOptions,
124    ) -> Result<Self> {
125        let mut revisions = manager.revisions();
126        let initial = manager.credential().await;
127        let names = AccountNames::derive(&initial.acct)?;
128        let (connection, mut ended) = loop {
129            revisions.borrow_and_update();
130            let credential = manager.credential().await;
131            match connect_current(manager.clone(), &credential, opts.request_timeout).await {
132                Ok(current) if !revisions.has_changed().unwrap_or(false) => break current,
133                Ok((connection, _)) => {
134                    let _ = connection.close(Duration::from_millis(50)).await;
135                }
136                Err(_)
137                    if manager.credential().await.credential_public
138                        != credential.credential_public => {}
139                Err(error) => return Err(error),
140            }
141        };
142        let connection = Arc::new(RwLock::new(connection));
143        let shared = connection.clone();
144        let (changes, receiver) = watch::channel(0);
145        let task = tokio::spawn(async move {
146            let mut retry = None;
147            loop {
148                let credential = manager.credential().await;
149                let delay = retry
150                    .take()
151                    .unwrap_or_else(|| credentials::renewal_delay(credential.exp));
152                tokio::select! {
153                    result = revisions.changed() => { if result.is_err() { return; } }
154                    () = connection_ended(&mut ended) => {
155                        // Replace it through the same path as a re-issue, so
156                        // event readers and bucket watches rebind to the new one.
157                        tracing::warn!("bus connection closed for good; replacing it");
158                    }
159                    () = tokio::time::sleep(delay) => {
160                        if let Err(error) = manager.renew().await {
161                            if credentials::transient(&error) || matches!(error, Error::Transport(_)) {
162                                retry = Some(Duration::from_secs(5));
163                                continue;
164                            }
165                            // A terminal refusal (notably generation_not_live)
166                            // must not cause issuance under a dead generation.
167                            let refusal_code = match &error {
168                                Error::Refused(code) => code.as_str(),
169                                _ => "client_renewal_error",
170                            };
171                            tracing::warn!(
172                                refusal_code,
173                                "bus credential maintenance stopped; reconnect or report degraded"
174                            );
175                            return;
176                        }
177                        if !revisions.has_changed().unwrap_or(false) { continue; }
178                    }
179                }
180                let mut backoff = Duration::from_millis(250);
181                loop {
182                    let revision = *revisions.borrow_and_update();
183                    let credential = manager.credential().await;
184                    match connect_current(manager.clone(), &credential, opts.request_timeout).await
185                    {
186                        Ok((replacement, replacement_ended)) => {
187                            if revisions.has_changed().unwrap_or(false) {
188                                let _ = replacement.close(Duration::from_millis(50)).await;
189                                continue;
190                            }
191                            let old = {
192                                let mut slot = shared
193                                    .write()
194                                    .unwrap_or_else(std::sync::PoisonError::into_inner);
195                                std::mem::replace(&mut *slot, replacement)
196                            };
197                            ended = replacement_ended;
198                            changes.send_replace(revision);
199                            // A replaced connection that is reconnecting can't
200                            // drain, and its auth callback now refuses every
201                            // attempt; MAX_RECONNECTS ends it instead.
202                            let _ = old.close(Duration::from_millis(50)).await;
203                            break;
204                        }
205                        Err(_) => {
206                            tokio::time::sleep(backoff).await;
207                            backoff = (backoff * 2).min(Duration::from_secs(30));
208                        }
209                    }
210                }
211            }
212        });
213        Ok(Self {
214            connection,
215            names,
216            module: opts.module,
217            changes: receiver,
218            task,
219        })
220    }
221
222    /// Returns a snapshot. A re-issue does not change an already held snapshot.
223    pub fn connection(&self) -> NatsConnection {
224        self.connection
225            .read()
226            .unwrap_or_else(std::sync::PoisonError::into_inner)
227            .clone()
228    }
229
230    /// Changes only after a replacement connection with a new inbox is ready.
231    /// Reopen raw subscriptions and durable pulls when this receiver changes;
232    /// [`EventReader`] handles its own rebinding automatically.
233    ///
234    /// If credential maintenance can no longer renew or re-issue, its task ends
235    /// and drops the sender. The receiver's `changed()` then returns `Err` once
236    /// any unseen change has been consumed. Reconnect with [`BusClient::connect`]
237    /// or report the module as degraded when the change stream closes.
238    pub fn connection_changes(&self) -> watch::Receiver<u64> {
239        self.changes.clone()
240    }
241
242    pub fn acct(&self) -> &str {
243        self.names.account()
244    }
245
246    /// Attaches to an existing EVENT durable owned by this module.
247    ///
248    /// ck-bus provisions durables; this method never creates or changes one.
249    /// The name must follow bus-naming's `m_{module}[_suffix]` owner rule.
250    /// The reader rebinds automatically after credential re-issue. Nothing is
251    /// acknowledged automatically: commit your own transaction before `ack()`.
252    pub async fn event_reader(&self, durable: &str) -> Result<EventReader> {
253        EventReader::attach(self, durable).await
254    }
255
256    /// Opens a declared module bucket for its owner or a declared reader.
257    /// Does not provision buckets or change their limits.
258    pub fn bucket(&self, owner_module: &str, bucket: &str) -> Result<Bucket> {
259        Bucket::attach(self, owner_module, bucket)
260    }
261
262    /// Stores an event using the fact's own id for JetStream deduplication.
263    ///
264    /// Event bodies carry ids, counters and enums, not large content. Bodies
265    /// larger than 4 KiB are refused; retrieve large content through its owner's
266    /// own call instead.
267    /// Inline bodies reach every reader whose durable matches the subject, with
268    /// no per-reader visibility check; reserve them for pointers and facts every
269    /// declared reader may see. Flow triggers must use [`Self::publish_claim_check`],
270    /// the `cortexkit_bus_trait::Stream::publish` claim-check path (empty payload plus
271    /// content digest), with the publisher serving the body under the reading
272    /// flow's own scope to check which agent may see it. The flow engine refuses
273    /// inline trigger bodies.
274    pub async fn publish_event(
275        &self,
276        event: &str,
277        version: u32,
278        qualifier: Option<&str>,
279        msg_id: &str,
280        body: impl Into<Bytes>,
281    ) -> Result<PublishReceipt> {
282        let body = body.into();
283        if body.len() > 4096 {
284            return Err(Error::EventBodyTooLarge { size: body.len() });
285        }
286        let subject = self.event_subject(event, version, qualifier)?;
287        let mut headers = async_nats::HeaderMap::new();
288        headers.insert(async_nats::header::NATS_MESSAGE_ID, msg_id);
289        let ack = async_nats::jetstream::new(self.connection().client())
290            .publish_with_headers(subject, headers, body)
291            .await
292            .map_err(|error| Error::Nats(error.to_string()))?
293            .await
294            .map_err(|error| Error::Nats(error.to_string()))?;
295        Ok(PublishReceipt::new(ack.sequence, ack.duplicate))
296    }
297
298    /// Publishes an empty-payload claim check carrying the body's content digest.
299    /// Store the body before publishing and serve it through the publisher's own
300    /// `events_get` query under the reading flow's scope. This is the event shape
301    /// for triggering flows; inline bodies cannot be used as flow triggers.
302    /// The NATS codec writes Nats-Msg-Id and Ck-Content-Digest and refuses caller
303    /// headers spelling either reserved name, including case variants.
304    #[allow(clippy::too_many_arguments)]
305    pub async fn publish_claim_check(
306        &self,
307        event: &str,
308        version: u32,
309        qualifier: Option<&str>,
310        msg_id: &str,
311        digest: ContentDigest,
312        headers: Headers,
313    ) -> Result<PublishReceipt> {
314        let subject = self.event_subject(event, version, qualifier)?;
315        self.connection()
316            .stream(&self.names.streams().event)
317            .publish(&subject, msg_id, digest, headers)
318            .await
319            .map_err(Error::from)
320    }
321
322    fn event_subject(&self, event: &str, version: u32, qualifier: Option<&str>) -> Result<String> {
323        Ok(match qualifier {
324            Some(reader) => {
325                self.names
326                    .event_subject_with_reader(&self.module, event, version, reader)?
327            }
328            None => self.names.event_subject(&self.module, event, version)?,
329        })
330    }
331
332    /// Publishes an ephemeral, versioned hint, not a stored fact.
333    pub async fn live_hint(&self, kind: &str, version: u32, body: impl Into<Bytes>) -> Result<()> {
334        let subject = self.names.live_subject(&self.module, kind, version)?;
335        self.connection()
336            .client()
337            .publish(subject, body.into())
338            .await
339            .map_err(|error| Error::Nats(error.to_string()))
340    }
341}
342
343/// How many reconnect attempts async-nats makes after a connection drops before
344/// it gives up and ends the connection for good.
345///
346/// async-nats waits nothing before the first attempt, then 2 ms doubling up to
347/// a 4 s cap, so twenty attempts span about 36 s: a ck-bus restart of about
348/// 30 s rides through on the same connection. The bound matters most for a
349/// replaced connection. Its credential is no longer current, so its auth
350/// callback refuses every attempt; without a bound it would keep connecting
351/// and failing every 4 s for the life of the process.
352const MAX_RECONNECTS: usize = 20;
353
354/// A subject under the connection's own inbox that nothing ever publishes to.
355/// A hyphen never appears in async-nats's generated inbox tokens, so it
356/// cannot collide with a request inbox.
357pub(crate) const LIVENESS_SUBJECT: &str = "bus-client-liveness";
358
359/// Resolves once async-nats has ended the connection for good: after it runs
360/// out of reconnect attempts, or after a drain. The connection's handler task
361/// then exits and drops every subscription, which ends this stream. The
362/// subscription survives ordinary reconnects because async-nats resubscribes it.
363/// async-nats 0.50 emits no `Event::Closed` when it gives up reconnecting, only
364/// a `ClientError::MaxReconnects` event that can be dropped from a full queue,
365/// and `NatsConnection` keeps its event stream private; the ended subscription
366/// is the signal that can't be missed.
367async fn connection_ended(ended: &mut async_nats::Subscriber) {
368    use futures_util::StreamExt;
369    while ended.next().await.is_some() {}
370}
371
372async fn connect_current(
373    manager: Arc<CredentialManager>,
374    credential: &credentials::Credential,
375    timeout: Duration,
376) -> Result<(NatsConnection, async_nats::Subscriber)> {
377    let public = credential.credential_public.clone();
378    let options = async_nats::ConnectOptions::with_auth_callback(move |nonce| {
379        let manager = manager.clone();
380        let public = public.clone();
381        // The spawned subc request future only needs to be Send; async-nats
382        // also requires the callback future to be Sync.
383        let task = tokio::spawn(async move {
384            let credential = manager.credential().await;
385            if credential.credential_public != public {
386                return Err(Error::Reply("connection credential was replaced"));
387            }
388            let signature = manager.sign(&public, nonce).await?;
389            let mut auth = async_nats::Auth::new();
390            auth.jwt = Some(credential.jwt);
391            auth.signature = Some(signature);
392            Ok(auth)
393        });
394        async move {
395            task.await
396                .map_err(async_nats::AuthError::new)?
397                .map_err(async_nats::AuthError::new)
398        }
399    })
400    .connection_timeout(timeout)
401    .request_timeout(Some(timeout))
402    .max_reconnects(MAX_RECONNECTS);
403    let mut config = ConnectConfig::new(&credential.credential_public)?;
404    config.request_timeout = timeout;
405    let connection = NatsConnection::connect(credential.server_url.as_str(), options, config)
406        .await
407        .map_err(Error::from)?;
408    let subject = format!("{}.{LIVENESS_SUBJECT}", connection.inbox_prefix());
409    match connection.client().subscribe(subject).await {
410        Ok(ended) => Ok((connection, ended)),
411        Err(error) => {
412            let _ = connection.close(Duration::from_millis(50)).await;
413            Err(Error::Nats(error.to_string()))
414        }
415    }
416}
417
418#[cfg(test)]
419mod tests;