Skip to main content

cortexkit_bus_client/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod binding;
4mod buckets;
5mod credentials;
6mod events;
7mod hub;
8
9pub use buckets::{Bucket, BucketEntry, BucketItem, BucketWatch};
10pub use events::{EventDelivery, EventItem, EventReader, MalformedDelivery, MalformedReason};
11pub use hub::HubClient;
12
13use std::{
14    sync::{Arc, RwLock},
15    time::Duration,
16};
17
18use bytes::Bytes;
19use cortexkit_bus_naming::{validate_token, AccountNames, TokenKind};
20use cortexkit_bus_nats::{ConnectConfig, NatsConnection};
21use cortexkit_bus_trait::{ContentDigest, Headers, PublishReceipt, Stream};
22
23/// The header carrying a body's SHA-256, spelled as the NATS codec writes it
24/// for claim checks.
25const CONTENT_DIGEST_HEADER: &str = "Ck-Content-Digest";
26use subc_client_rs::SubcConsumer;
27use tokio::sync::watch;
28
29use credentials::CredentialManager;
30
31/// Errors contain refusal codes, not credential JWTs or nonce signatures.
32#[derive(Debug, thiserror::Error)]
33#[non_exhaustive]
34pub enum Error {
35    #[error("ck-bus refused the request: {0}")]
36    Refused(String),
37    /// Retry when the module next needs the hub.
38    #[error("ck-bus refused the hub credential: hub_unavailable")]
39    HubUnavailable,
40    /// This module is not allowed to use the hub. Do not retry.
41    #[error("ck-bus refused the hub credential: hub_not_granted")]
42    HubNotGranted,
43    #[error("ck-bus request failed: {0}")]
44    Transport(String),
45    #[error("invalid ck-bus reply: {0}")]
46    Reply(&'static str),
47    #[error(transparent)]
48    Naming(#[from] cortexkit_bus_naming::NamingError),
49    #[error(transparent)]
50    Bus(#[from] cortexkit_bus_trait::BusError),
51    #[error("NATS operation failed: {0}")]
52    Nats(String),
53    #[error("event body is {size} bytes; the maximum is 4096 bytes")]
54    EventBodyTooLarge { size: usize },
55    #[error("durable {durable} is not owned by module {module}")]
56    DurableNotOwned { durable: String, module: String },
57    #[error("module bucket {owner}.{bucket} is not declared")]
58    BucketNotDeclared { owner: String, bucket: String },
59    #[error("module {module} may not read bucket {bucket}")]
60    BucketNotReadable { module: String, bucket: String },
61    #[error("module {module} does not own bucket {bucket}")]
62    BucketNotOwned { module: String, bucket: String },
63    #[error("bucket value is {size} bytes; the maximum is {cap} bytes")]
64    BucketValueTooLarge { size: usize, cap: u32 },
65    #[error("bucket {bucket} has no watch consumer slots available")]
66    WatchCapacity { bucket: String },
67}
68
69pub type Result<T> = std::result::Result<T, Error>;
70
71/// Options for one module's bus connection.
72#[derive(Clone, Debug)]
73#[non_exhaustive]
74pub struct BusClientOptions {
75    module: String,
76    pub request_timeout: Duration,
77}
78
79impl BusClientOptions {
80    pub fn new(module: impl Into<String>) -> Result<Self> {
81        let module = module.into();
82        validate_token(TokenKind::ModuleId, &module)?;
83        Ok(Self {
84            module,
85            request_timeout: Duration::from_secs(5),
86        })
87    }
88}
89
90/// A module's current connection and the task that maintains its credentials.
91pub struct BusClient {
92    managed: ManagedConnection,
93    names: AccountNames,
94    module: String,
95}
96
97struct ManagedConnection {
98    connection: Arc<RwLock<NatsConnection>>,
99    changes: watch::Receiver<u64>,
100    task: tokio::task::JoinHandle<()>,
101}
102
103impl Drop for ManagedConnection {
104    fn drop(&mut self) {
105        self.task.abort();
106    }
107}
108
109impl BusClient {
110    /// Issues a credential for the module identity that the subc daemon
111    /// authenticates from the module's launch nonce.
112    pub async fn connect(subc: Arc<SubcConsumer>, opts: BusClientOptions) -> Result<Self> {
113        let manager = credentials::module_manager(subc, &opts, false).await?;
114        Self::connect_manager(manager, opts).await
115    }
116
117    async fn connect_manager(
118        manager: Arc<CredentialManager>,
119        opts: BusClientOptions,
120    ) -> Result<Self> {
121        let initial = manager.credential().await;
122        let names = AccountNames::derive(&initial.acct)?;
123        let managed = ManagedConnection::connect(manager, opts.request_timeout).await?;
124        Ok(Self {
125            managed,
126            names,
127            module: opts.module,
128        })
129    }
130}
131
132impl ManagedConnection {
133    async fn connect(manager: Arc<CredentialManager>, timeout: Duration) -> Result<Self> {
134        let mut revisions = manager.revisions();
135        let (connection, mut ended) = loop {
136            revisions.borrow_and_update();
137            manager.check_hub_refusal()?;
138            let credential = manager.credential().await;
139            match connect_current(manager.clone(), &credential, timeout).await {
140                Ok(current) if !revisions.has_changed().unwrap_or(false) => break current,
141                Ok((connection, _)) => {
142                    let _ = connection.close(Duration::from_millis(50)).await;
143                }
144                Err(_)
145                    if manager.credential().await.credential_public
146                        != credential.credential_public => {}
147                Err(error) => {
148                    manager.check_hub_refusal()?;
149                    return Err(error);
150                }
151            }
152        };
153        let connection = Arc::new(RwLock::new(connection));
154        let shared = connection.clone();
155        let (changes, receiver) = watch::channel(0);
156        let task = tokio::spawn(async move {
157            let mut retry = None;
158            loop {
159                if manager.check_hub_refusal().is_err() {
160                    let current = shared
161                        .read()
162                        .unwrap_or_else(std::sync::PoisonError::into_inner)
163                        .clone();
164                    let _ = current.close(Duration::from_millis(50)).await;
165                    return;
166                }
167                let credential = manager.credential().await;
168                let delay = retry
169                    .take()
170                    .unwrap_or_else(|| credentials::renewal_delay(credential.exp));
171                tokio::select! {
172                    result = revisions.changed() => { if result.is_err() { return; } }
173                    () = connection_ended(&mut ended) => {
174                        // Replace it through the same path as a re-issue, so
175                        // event readers and bucket watches rebind to the new one.
176                        tracing::warn!("bus connection closed for good; replacing it");
177                    }
178                    () = tokio::time::sleep(delay) => {
179                        if let Err(error) = manager.renew().await {
180                            if matches!(error, Error::HubUnavailable | Error::HubNotGranted) {
181                                continue;
182                            }
183                            if credentials::transient(&error) || matches!(error, Error::Transport(_)) {
184                                retry = Some(Duration::from_secs(5));
185                                continue;
186                            }
187                            // A terminal refusal (notably generation_not_live)
188                            // must not cause issuance under a dead generation.
189                            let refusal_code = match &error {
190                                Error::Refused(code) => code.as_str(),
191                                _ => "client_renewal_error",
192                            };
193                            tracing::warn!(
194                                refusal_code,
195                                "bus credential maintenance stopped; reconnect or report degraded"
196                            );
197                            return;
198                        }
199                        if !revisions.has_changed().unwrap_or(false) { continue; }
200                    }
201                }
202                let mut backoff = Duration::from_millis(250);
203                loop {
204                    let revision = *revisions.borrow_and_update();
205                    if manager.check_hub_refusal().is_err() {
206                        break;
207                    }
208                    let credential = manager.credential().await;
209                    match connect_current(manager.clone(), &credential, timeout).await {
210                        Ok((replacement, replacement_ended)) => {
211                            if revisions.has_changed().unwrap_or(false) {
212                                let _ = replacement.close(Duration::from_millis(50)).await;
213                                continue;
214                            }
215                            let old = {
216                                let mut slot = shared
217                                    .write()
218                                    .unwrap_or_else(std::sync::PoisonError::into_inner);
219                                std::mem::replace(&mut *slot, replacement)
220                            };
221                            ended = replacement_ended;
222                            changes.send_replace(revision);
223                            // A replaced connection that is reconnecting can't
224                            // drain, and its auth callback now refuses every
225                            // attempt; MAX_RECONNECTS ends it instead.
226                            let _ = old.close(Duration::from_millis(50)).await;
227                            break;
228                        }
229                        Err(_) => {
230                            if manager.check_hub_refusal().is_err() {
231                                break;
232                            }
233                            tokio::time::sleep(backoff).await;
234                            backoff = (backoff * 2).min(Duration::from_secs(30));
235                        }
236                    }
237                }
238            }
239        });
240        Ok(Self {
241            connection,
242            changes: receiver,
243            task,
244        })
245    }
246
247    fn snapshot(&self) -> NatsConnection {
248        self.connection
249            .read()
250            .unwrap_or_else(std::sync::PoisonError::into_inner)
251            .clone()
252    }
253}
254
255impl BusClient {
256    /// Returns a snapshot. A re-issue does not change an already held snapshot.
257    pub fn connection(&self) -> NatsConnection {
258        self.managed.snapshot()
259    }
260
261    /// Changes only after a replacement connection with its issued inbox prefix is ready.
262    /// Reopen raw subscriptions and durable pulls when this receiver changes;
263    /// [`EventReader`] handles its own rebinding automatically.
264    ///
265    /// If credential maintenance can no longer renew or re-issue, its task ends
266    /// and drops the sender. The receiver's `changed()` then returns `Err` once
267    /// any unseen change has been consumed. Reconnect with [`BusClient::connect`]
268    /// or report the module as degraded when the change stream closes.
269    pub fn connection_changes(&self) -> watch::Receiver<u64> {
270        self.managed.changes.clone()
271    }
272
273    pub fn acct(&self) -> &str {
274        self.names.account()
275    }
276
277    /// Returns the computer's box id as supplied by the bus module's account.
278    ///
279    /// Modules must never derive this id themselves. Returns `None` unless the
280    /// account is `box_` followed by a canonical 32-character box id.
281    pub fn box_id(&self) -> Option<cortexkit_bus_naming::federation::BoxId> {
282        credentials::box_id(self.acct())
283    }
284
285    /// Returns the inbox prefix used by the current connection.
286    ///
287    /// A per-credential prefix changes after credential re-issue; a module's
288    /// `ckdel` prefix stays stable. Use [`Self::connection_changes`] to observe
289    /// replacements, or [`NatsConnection::inbox_prefix`] for a held snapshot.
290    pub fn inbox_prefix(&self) -> String {
291        self.connection().inbox_prefix().to_owned()
292    }
293
294    /// Attaches to an existing EVENT durable owned by this module.
295    ///
296    /// ck-bus provisions durables; this method never creates or changes one.
297    /// The name must follow bus-naming's `m_{module}[_suffix]` owner rule.
298    /// The reader rebinds automatically after credential re-issue. Nothing is
299    /// acknowledged automatically: commit your own transaction before `ack()`.
300    pub async fn event_reader(&self, durable: &str) -> Result<EventReader> {
301        EventReader::attach(self, durable).await
302    }
303
304    /// Opens a declared module bucket for its owner or a declared reader.
305    /// Does not provision buckets or change their limits.
306    pub fn bucket(&self, owner_module: &str, bucket: &str) -> Result<Bucket> {
307        Bucket::attach(self, owner_module, bucket)
308    }
309
310    /// Stores an event using the fact's own id for JetStream deduplication.
311    ///
312    /// Event bodies carry ids, counters and enums, not large content. Bodies
313    /// larger than 4 KiB are refused; retrieve large content through its owner's
314    /// own call instead.
315    /// A non-empty body is sent with its SHA-256 as `Ck-Content-Digest`, so a
316    /// reader can check the bytes it received. An empty body is sent without one,
317    /// because an empty payload with a digest is the claim-check shape.
318    /// Inline bodies reach every reader whose durable matches the subject, with
319    /// no per-reader visibility check; reserve them for pointers and facts every
320    /// declared reader may see. Flow triggers must use [`Self::publish_claim_check`],
321    /// the `cortexkit_bus_trait::Stream::publish` claim-check path (empty payload plus
322    /// content digest), with the publisher serving the body under the reading
323    /// flow's own scope to check which agent may see it. The flow engine refuses
324    /// inline trigger bodies.
325    pub async fn publish_event(
326        &self,
327        event: &str,
328        version: u32,
329        qualifier: Option<&str>,
330        msg_id: &str,
331        body: impl Into<Bytes>,
332    ) -> Result<PublishReceipt> {
333        let body = body.into();
334        if body.len() > 4096 {
335            return Err(Error::EventBodyTooLarge { size: body.len() });
336        }
337        let subject = self.event_subject(event, version, qualifier)?;
338        let mut headers = async_nats::HeaderMap::new();
339        headers.insert(async_nats::header::NATS_MESSAGE_ID, msg_id);
340        if !body.is_empty() {
341            headers.insert(
342                CONTENT_DIGEST_HEADER,
343                ContentDigest::of_bytes(&body).to_string().as_str(),
344            );
345        }
346        let ack = async_nats::jetstream::new(self.connection().client())
347            .publish_with_headers(subject, headers, body)
348            .await
349            .map_err(|error| Error::Nats(error.to_string()))?
350            .await
351            .map_err(|error| Error::Nats(error.to_string()))?;
352        Ok(PublishReceipt::new(ack.sequence, ack.duplicate))
353    }
354
355    /// Publishes an empty-payload claim check carrying the body's content digest.
356    /// Store the body before publishing and serve it through the publisher's own
357    /// `events_get` query under the reading flow's scope. This is the event shape
358    /// for triggering flows; inline bodies cannot be used as flow triggers.
359    /// The NATS codec writes Nats-Msg-Id and Ck-Content-Digest and refuses caller
360    /// headers spelling either reserved name, including case variants.
361    #[allow(clippy::too_many_arguments)]
362    pub async fn publish_claim_check(
363        &self,
364        event: &str,
365        version: u32,
366        qualifier: Option<&str>,
367        msg_id: &str,
368        digest: ContentDigest,
369        headers: Headers,
370    ) -> Result<PublishReceipt> {
371        let subject = self.event_subject(event, version, qualifier)?;
372        self.connection()
373            .stream(&self.names.streams().event)
374            .publish(&subject, msg_id, digest, headers)
375            .await
376            .map_err(Error::from)
377    }
378
379    fn event_subject(&self, event: &str, version: u32, qualifier: Option<&str>) -> Result<String> {
380        Ok(match qualifier {
381            Some(reader) => {
382                self.names
383                    .event_subject_with_reader(&self.module, event, version, reader)?
384            }
385            None => self.names.event_subject(&self.module, event, version)?,
386        })
387    }
388
389    /// Publishes an ephemeral, versioned hint, not a stored fact.
390    pub async fn live_hint(&self, kind: &str, version: u32, body: impl Into<Bytes>) -> Result<()> {
391        let subject = self.names.live_subject(&self.module, kind, version)?;
392        self.connection()
393            .client()
394            .publish(subject, body.into())
395            .await
396            .map_err(|error| Error::Nats(error.to_string()))
397    }
398}
399
400/// How many reconnect attempts async-nats makes after a connection drops before
401/// it gives up and ends the connection for good.
402///
403/// async-nats waits nothing before the first attempt, then 2 ms doubling up to
404/// a 4 s cap, so twenty attempts span about 36 s: a ck-bus restart of about
405/// 30 s rides through on the same connection. The bound matters most for a
406/// replaced connection. Its credential is no longer current, so its auth
407/// callback refuses every attempt; without a bound it would keep connecting
408/// and failing every 4 s for the life of the process.
409const MAX_RECONNECTS: usize = 20;
410
411/// A subject under the connection's own inbox that nothing ever publishes to.
412/// A hyphen never appears in async-nats's generated inbox tokens, so it
413/// cannot collide with a request inbox.
414pub(crate) const LIVENESS_SUBJECT: &str = "bus-client-liveness";
415
416/// Resolves once async-nats has ended the connection for good: after it runs
417/// out of reconnect attempts, or after a drain. The connection's handler task
418/// then exits and drops every subscription, which ends this stream. The
419/// subscription survives ordinary reconnects because async-nats resubscribes it.
420/// async-nats 0.50 emits no `Event::Closed` when it gives up reconnecting, only
421/// a `ClientError::MaxReconnects` event that can be dropped from a full queue,
422/// and `NatsConnection` keeps its event stream private; the ended subscription
423/// is the signal that can't be missed.
424async fn connection_ended(ended: &mut async_nats::Subscriber) {
425    use futures_util::StreamExt;
426    while ended.next().await.is_some() {}
427}
428
429async fn connect_current(
430    manager: Arc<CredentialManager>,
431    credential: &credentials::Credential,
432    timeout: Duration,
433) -> Result<(NatsConnection, async_nats::Subscriber)> {
434    let public = credential.credential_public.clone();
435    let options = async_nats::ConnectOptions::with_auth_callback(move |nonce| {
436        let manager = manager.clone();
437        let public = public.clone();
438        // The spawned subc request future only needs to be Send; async-nats
439        // also requires the callback future to be Sync.
440        let task = tokio::spawn(async move {
441            let credential = manager.credential().await;
442            if credential.credential_public != public {
443                return Err(Error::Reply("connection credential was replaced"));
444            }
445            let signature = manager.sign(&public, nonce).await?;
446            let mut auth = async_nats::Auth::new();
447            auth.jwt = Some(credential.jwt);
448            auth.signature = Some(signature);
449            Ok(auth)
450        });
451        async move {
452            task.await
453                .map_err(async_nats::AuthError::new)?
454                .map_err(async_nats::AuthError::new)
455        }
456    })
457    .connection_timeout(timeout)
458    .request_timeout(Some(timeout))
459    .max_reconnects(MAX_RECONNECTS);
460    let mut config = ConnectConfig::new(&credential.credential_public)?;
461    config.request_timeout = timeout;
462    let connection = if credential.inbox_prefix == config.inbox_prefix() {
463        NatsConnection::connect(credential.server_url.as_str(), options, config).await
464    } else {
465        NatsConnection::connect_with_inbox_prefix(
466            credential.server_url.as_str(),
467            options,
468            config,
469            &credential.inbox_prefix,
470        )
471        .await
472    }
473    .map_err(Error::from)?;
474    let subject = format!("{}.{LIVENESS_SUBJECT}", connection.inbox_prefix());
475    match connection.client().subscribe(subject).await {
476        Ok(ended) => Ok((connection, ended)),
477        Err(error) => {
478            let _ = connection.close(Duration::from_millis(50)).await;
479            Err(Error::Nats(error.to_string()))
480        }
481    }
482}
483
484#[cfg(test)]
485mod tests;