Skip to main content

cortexkit_bus_client/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod binding;
4mod buckets;
5mod credentials;
6mod events;
7
8pub use buckets::{Bucket, BucketEntry, BucketItem, BucketWatch};
9pub use events::{EventDelivery, EventItem, EventReader, MalformedDelivery, MalformedReason};
10
11use std::{
12    sync::{Arc, RwLock},
13    time::Duration,
14};
15
16use bytes::Bytes;
17use cortexkit_bus_naming::{validate_token, AccountNames, TokenKind};
18use cortexkit_bus_nats::{ConnectConfig, NatsConnection};
19use cortexkit_bus_trait::{ContentDigest, Headers, PublishReceipt, Stream};
20
21/// The header carrying a body's SHA-256, spelled as the NATS codec writes it
22/// for claim checks.
23const CONTENT_DIGEST_HEADER: &str = "Ck-Content-Digest";
24use subc_client_rs::SubcConsumer;
25use tokio::sync::watch;
26
27use credentials::{CredentialManager, SubcTransport};
28
29/// Errors contain refusal codes, not credential JWTs or nonce signatures.
30#[derive(Debug, thiserror::Error)]
31#[non_exhaustive]
32pub enum Error {
33    #[error("ck-bus refused the request: {0}")]
34    Refused(String),
35    #[error("ck-bus request failed: {0}")]
36    Transport(String),
37    #[error("invalid ck-bus reply: {0}")]
38    Reply(&'static str),
39    #[error(transparent)]
40    Naming(#[from] cortexkit_bus_naming::NamingError),
41    #[error(transparent)]
42    Bus(#[from] cortexkit_bus_trait::BusError),
43    #[error("NATS operation failed: {0}")]
44    Nats(String),
45    #[error("event body is {size} bytes; the maximum is 4096 bytes")]
46    EventBodyTooLarge { size: usize },
47    #[error("durable {durable} is not owned by module {module}")]
48    DurableNotOwned { durable: String, module: String },
49    #[error("module bucket {owner}.{bucket} is not declared")]
50    BucketNotDeclared { owner: String, bucket: String },
51    #[error("module {module} may not read bucket {bucket}")]
52    BucketNotReadable { module: String, bucket: String },
53    #[error("module {module} does not own bucket {bucket}")]
54    BucketNotOwned { module: String, bucket: String },
55    #[error("bucket value is {size} bytes; the maximum is {cap} bytes")]
56    BucketValueTooLarge { size: usize, cap: u32 },
57    #[error("bucket {bucket} has no watch consumer slots available")]
58    WatchCapacity { bucket: String },
59}
60
61pub type Result<T> = std::result::Result<T, Error>;
62
63/// Options for one module's bus connection.
64#[derive(Clone, Debug)]
65#[non_exhaustive]
66pub struct BusClientOptions {
67    module: String,
68    pub request_timeout: Duration,
69}
70
71impl BusClientOptions {
72    pub fn new(module: impl Into<String>) -> Result<Self> {
73        let module = module.into();
74        validate_token(TokenKind::ModuleId, &module)?;
75        Ok(Self {
76            module,
77            request_timeout: Duration::from_secs(5),
78        })
79    }
80}
81
82/// A module's current connection and the task that maintains its credentials.
83pub struct BusClient {
84    connection: Arc<RwLock<NatsConnection>>,
85    names: AccountNames,
86    module: String,
87    changes: watch::Receiver<u64>,
88    task: tokio::task::JoinHandle<()>,
89}
90
91impl Drop for BusClient {
92    fn drop(&mut self) {
93        self.task.abort();
94    }
95}
96
97impl BusClient {
98    /// Issues a credential for the module identity that the subc daemon
99    /// authenticates from the module's launch nonce.
100    pub async fn connect(subc: Arc<SubcConsumer>, opts: BusClientOptions) -> Result<Self> {
101        let transport = SubcTransport::new(subc, &opts.module, opts.request_timeout);
102        // One module process can keep several connection snapshots, but issuing
103        // independently for them would revoke the other snapshots' credentials.
104        static MANAGERS: std::sync::LazyLock<
105            tokio::sync::Mutex<
106                std::collections::HashMap<String, std::sync::Weak<CredentialManager>>,
107            >,
108        > = std::sync::LazyLock::new(Default::default);
109        let mut managers = MANAGERS.lock().await;
110        let manager = match managers
111            .get(&opts.module)
112            .and_then(std::sync::Weak::upgrade)
113        {
114            Some(manager) => manager,
115            None => {
116                let manager = CredentialManager::start(Arc::new(transport)).await?;
117                managers.insert(opts.module.clone(), Arc::downgrade(&manager));
118                manager
119            }
120        };
121        drop(managers);
122        Self::connect_manager(manager, opts).await
123    }
124
125    async fn connect_manager(
126        manager: Arc<CredentialManager>,
127        opts: BusClientOptions,
128    ) -> Result<Self> {
129        let mut revisions = manager.revisions();
130        let initial = manager.credential().await;
131        let names = AccountNames::derive(&initial.acct)?;
132        let (connection, mut ended) = loop {
133            revisions.borrow_and_update();
134            let credential = manager.credential().await;
135            match connect_current(manager.clone(), &credential, opts.request_timeout).await {
136                Ok(current) if !revisions.has_changed().unwrap_or(false) => break current,
137                Ok((connection, _)) => {
138                    let _ = connection.close(Duration::from_millis(50)).await;
139                }
140                Err(_)
141                    if manager.credential().await.credential_public
142                        != credential.credential_public => {}
143                Err(error) => return Err(error),
144            }
145        };
146        let connection = Arc::new(RwLock::new(connection));
147        let shared = connection.clone();
148        let (changes, receiver) = watch::channel(0);
149        let task = tokio::spawn(async move {
150            let mut retry = None;
151            loop {
152                let credential = manager.credential().await;
153                let delay = retry
154                    .take()
155                    .unwrap_or_else(|| credentials::renewal_delay(credential.exp));
156                tokio::select! {
157                    result = revisions.changed() => { if result.is_err() { return; } }
158                    () = connection_ended(&mut ended) => {
159                        // Replace it through the same path as a re-issue, so
160                        // event readers and bucket watches rebind to the new one.
161                        tracing::warn!("bus connection closed for good; replacing it");
162                    }
163                    () = tokio::time::sleep(delay) => {
164                        if let Err(error) = manager.renew().await {
165                            if credentials::transient(&error) || matches!(error, Error::Transport(_)) {
166                                retry = Some(Duration::from_secs(5));
167                                continue;
168                            }
169                            // A terminal refusal (notably generation_not_live)
170                            // must not cause issuance under a dead generation.
171                            let refusal_code = match &error {
172                                Error::Refused(code) => code.as_str(),
173                                _ => "client_renewal_error",
174                            };
175                            tracing::warn!(
176                                refusal_code,
177                                "bus credential maintenance stopped; reconnect or report degraded"
178                            );
179                            return;
180                        }
181                        if !revisions.has_changed().unwrap_or(false) { continue; }
182                    }
183                }
184                let mut backoff = Duration::from_millis(250);
185                loop {
186                    let revision = *revisions.borrow_and_update();
187                    let credential = manager.credential().await;
188                    match connect_current(manager.clone(), &credential, opts.request_timeout).await
189                    {
190                        Ok((replacement, replacement_ended)) => {
191                            if revisions.has_changed().unwrap_or(false) {
192                                let _ = replacement.close(Duration::from_millis(50)).await;
193                                continue;
194                            }
195                            let old = {
196                                let mut slot = shared
197                                    .write()
198                                    .unwrap_or_else(std::sync::PoisonError::into_inner);
199                                std::mem::replace(&mut *slot, replacement)
200                            };
201                            ended = replacement_ended;
202                            changes.send_replace(revision);
203                            // A replaced connection that is reconnecting can't
204                            // drain, and its auth callback now refuses every
205                            // attempt; MAX_RECONNECTS ends it instead.
206                            let _ = old.close(Duration::from_millis(50)).await;
207                            break;
208                        }
209                        Err(_) => {
210                            tokio::time::sleep(backoff).await;
211                            backoff = (backoff * 2).min(Duration::from_secs(30));
212                        }
213                    }
214                }
215            }
216        });
217        Ok(Self {
218            connection,
219            names,
220            module: opts.module,
221            changes: receiver,
222            task,
223        })
224    }
225
226    /// Returns a snapshot. A re-issue does not change an already held snapshot.
227    pub fn connection(&self) -> NatsConnection {
228        self.connection
229            .read()
230            .unwrap_or_else(std::sync::PoisonError::into_inner)
231            .clone()
232    }
233
234    /// Changes only after a replacement connection with a new inbox is ready.
235    /// Reopen raw subscriptions and durable pulls when this receiver changes;
236    /// [`EventReader`] handles its own rebinding automatically.
237    ///
238    /// If credential maintenance can no longer renew or re-issue, its task ends
239    /// and drops the sender. The receiver's `changed()` then returns `Err` once
240    /// any unseen change has been consumed. Reconnect with [`BusClient::connect`]
241    /// or report the module as degraded when the change stream closes.
242    pub fn connection_changes(&self) -> watch::Receiver<u64> {
243        self.changes.clone()
244    }
245
246    pub fn acct(&self) -> &str {
247        self.names.account()
248    }
249
250    /// Attaches to an existing EVENT durable owned by this module.
251    ///
252    /// ck-bus provisions durables; this method never creates or changes one.
253    /// The name must follow bus-naming's `m_{module}[_suffix]` owner rule.
254    /// The reader rebinds automatically after credential re-issue. Nothing is
255    /// acknowledged automatically: commit your own transaction before `ack()`.
256    pub async fn event_reader(&self, durable: &str) -> Result<EventReader> {
257        EventReader::attach(self, durable).await
258    }
259
260    /// Opens a declared module bucket for its owner or a declared reader.
261    /// Does not provision buckets or change their limits.
262    pub fn bucket(&self, owner_module: &str, bucket: &str) -> Result<Bucket> {
263        Bucket::attach(self, owner_module, bucket)
264    }
265
266    /// Stores an event using the fact's own id for JetStream deduplication.
267    ///
268    /// Event bodies carry ids, counters and enums, not large content. Bodies
269    /// larger than 4 KiB are refused; retrieve large content through its owner's
270    /// own call instead.
271    /// A non-empty body is sent with its SHA-256 as `Ck-Content-Digest`, so a
272    /// reader can check the bytes it received. An empty body is sent without one,
273    /// because an empty payload with a digest is the claim-check shape.
274    /// Inline bodies reach every reader whose durable matches the subject, with
275    /// no per-reader visibility check; reserve them for pointers and facts every
276    /// declared reader may see. Flow triggers must use [`Self::publish_claim_check`],
277    /// the `cortexkit_bus_trait::Stream::publish` claim-check path (empty payload plus
278    /// content digest), with the publisher serving the body under the reading
279    /// flow's own scope to check which agent may see it. The flow engine refuses
280    /// inline trigger bodies.
281    pub async fn publish_event(
282        &self,
283        event: &str,
284        version: u32,
285        qualifier: Option<&str>,
286        msg_id: &str,
287        body: impl Into<Bytes>,
288    ) -> Result<PublishReceipt> {
289        let body = body.into();
290        if body.len() > 4096 {
291            return Err(Error::EventBodyTooLarge { size: body.len() });
292        }
293        let subject = self.event_subject(event, version, qualifier)?;
294        let mut headers = async_nats::HeaderMap::new();
295        headers.insert(async_nats::header::NATS_MESSAGE_ID, msg_id);
296        if !body.is_empty() {
297            headers.insert(
298                CONTENT_DIGEST_HEADER,
299                ContentDigest::of_bytes(&body).to_string().as_str(),
300            );
301        }
302        let ack = async_nats::jetstream::new(self.connection().client())
303            .publish_with_headers(subject, headers, body)
304            .await
305            .map_err(|error| Error::Nats(error.to_string()))?
306            .await
307            .map_err(|error| Error::Nats(error.to_string()))?;
308        Ok(PublishReceipt::new(ack.sequence, ack.duplicate))
309    }
310
311    /// Publishes an empty-payload claim check carrying the body's content digest.
312    /// Store the body before publishing and serve it through the publisher's own
313    /// `events_get` query under the reading flow's scope. This is the event shape
314    /// for triggering flows; inline bodies cannot be used as flow triggers.
315    /// The NATS codec writes Nats-Msg-Id and Ck-Content-Digest and refuses caller
316    /// headers spelling either reserved name, including case variants.
317    #[allow(clippy::too_many_arguments)]
318    pub async fn publish_claim_check(
319        &self,
320        event: &str,
321        version: u32,
322        qualifier: Option<&str>,
323        msg_id: &str,
324        digest: ContentDigest,
325        headers: Headers,
326    ) -> Result<PublishReceipt> {
327        let subject = self.event_subject(event, version, qualifier)?;
328        self.connection()
329            .stream(&self.names.streams().event)
330            .publish(&subject, msg_id, digest, headers)
331            .await
332            .map_err(Error::from)
333    }
334
335    fn event_subject(&self, event: &str, version: u32, qualifier: Option<&str>) -> Result<String> {
336        Ok(match qualifier {
337            Some(reader) => {
338                self.names
339                    .event_subject_with_reader(&self.module, event, version, reader)?
340            }
341            None => self.names.event_subject(&self.module, event, version)?,
342        })
343    }
344
345    /// Publishes an ephemeral, versioned hint, not a stored fact.
346    pub async fn live_hint(&self, kind: &str, version: u32, body: impl Into<Bytes>) -> Result<()> {
347        let subject = self.names.live_subject(&self.module, kind, version)?;
348        self.connection()
349            .client()
350            .publish(subject, body.into())
351            .await
352            .map_err(|error| Error::Nats(error.to_string()))
353    }
354}
355
356/// How many reconnect attempts async-nats makes after a connection drops before
357/// it gives up and ends the connection for good.
358///
359/// async-nats waits nothing before the first attempt, then 2 ms doubling up to
360/// a 4 s cap, so twenty attempts span about 36 s: a ck-bus restart of about
361/// 30 s rides through on the same connection. The bound matters most for a
362/// replaced connection. Its credential is no longer current, so its auth
363/// callback refuses every attempt; without a bound it would keep connecting
364/// and failing every 4 s for the life of the process.
365const MAX_RECONNECTS: usize = 20;
366
367/// A subject under the connection's own inbox that nothing ever publishes to.
368/// A hyphen never appears in async-nats's generated inbox tokens, so it
369/// cannot collide with a request inbox.
370pub(crate) const LIVENESS_SUBJECT: &str = "bus-client-liveness";
371
372/// Resolves once async-nats has ended the connection for good: after it runs
373/// out of reconnect attempts, or after a drain. The connection's handler task
374/// then exits and drops every subscription, which ends this stream. The
375/// subscription survives ordinary reconnects because async-nats resubscribes it.
376/// async-nats 0.50 emits no `Event::Closed` when it gives up reconnecting, only
377/// a `ClientError::MaxReconnects` event that can be dropped from a full queue,
378/// and `NatsConnection` keeps its event stream private; the ended subscription
379/// is the signal that can't be missed.
380async fn connection_ended(ended: &mut async_nats::Subscriber) {
381    use futures_util::StreamExt;
382    while ended.next().await.is_some() {}
383}
384
385async fn connect_current(
386    manager: Arc<CredentialManager>,
387    credential: &credentials::Credential,
388    timeout: Duration,
389) -> Result<(NatsConnection, async_nats::Subscriber)> {
390    let public = credential.credential_public.clone();
391    let options = async_nats::ConnectOptions::with_auth_callback(move |nonce| {
392        let manager = manager.clone();
393        let public = public.clone();
394        // The spawned subc request future only needs to be Send; async-nats
395        // also requires the callback future to be Sync.
396        let task = tokio::spawn(async move {
397            let credential = manager.credential().await;
398            if credential.credential_public != public {
399                return Err(Error::Reply("connection credential was replaced"));
400            }
401            let signature = manager.sign(&public, nonce).await?;
402            let mut auth = async_nats::Auth::new();
403            auth.jwt = Some(credential.jwt);
404            auth.signature = Some(signature);
405            Ok(auth)
406        });
407        async move {
408            task.await
409                .map_err(async_nats::AuthError::new)?
410                .map_err(async_nats::AuthError::new)
411        }
412    })
413    .connection_timeout(timeout)
414    .request_timeout(Some(timeout))
415    .max_reconnects(MAX_RECONNECTS);
416    let mut config = ConnectConfig::new(&credential.credential_public)?;
417    config.request_timeout = timeout;
418    let connection = NatsConnection::connect(credential.server_url.as_str(), options, config)
419        .await
420        .map_err(Error::from)?;
421    let subject = format!("{}.{LIVENESS_SUBJECT}", connection.inbox_prefix());
422    match connection.client().subscribe(subject).await {
423        Ok(ended) => Ok((connection, ended)),
424        Err(error) => {
425            let _ = connection.close(Duration::from_millis(50)).await;
426            Err(Error::Nats(error.to_string()))
427        }
428    }
429}
430
431#[cfg(test)]
432mod tests;