Skip to main content

cortexkit_bus_client/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod binding;
4mod buckets;
5mod credentials;
6mod events;
7
8pub use buckets::{Bucket, BucketEntry, BucketItem, BucketWatch};
9pub use events::{EventDelivery, EventItem, EventReader, MalformedDelivery, MalformedReason};
10
11use std::{
12    sync::{Arc, RwLock},
13    time::Duration,
14};
15
16use bytes::Bytes;
17use cortexkit_bus_naming::{validate_token, AccountNames, TokenKind};
18use cortexkit_bus_nats::{ConnectConfig, NatsConnection};
19use cortexkit_bus_trait::{ContentDigest, Headers, PublishReceipt, Stream};
20
21/// The header carrying a body's SHA-256, spelled as the NATS codec writes it
22/// for claim checks.
23const CONTENT_DIGEST_HEADER: &str = "Ck-Content-Digest";
24use subc_client_rs::SubcConsumer;
25use tokio::sync::watch;
26
27use credentials::{CredentialManager, SubcTransport};
28
29/// Errors contain refusal codes, not credential JWTs or nonce signatures.
30#[derive(Debug, thiserror::Error)]
31#[non_exhaustive]
32pub enum Error {
33    #[error("ck-bus refused the request: {0}")]
34    Refused(String),
35    #[error("ck-bus request failed: {0}")]
36    Transport(String),
37    #[error("invalid ck-bus reply: {0}")]
38    Reply(&'static str),
39    #[error(transparent)]
40    Naming(#[from] cortexkit_bus_naming::NamingError),
41    #[error(transparent)]
42    Bus(#[from] cortexkit_bus_trait::BusError),
43    #[error("NATS operation failed: {0}")]
44    Nats(String),
45    #[error("event body is {size} bytes; the maximum is 4096 bytes")]
46    EventBodyTooLarge { size: usize },
47    #[error("durable {durable} is not owned by module {module}")]
48    DurableNotOwned { durable: String, module: String },
49    #[error("module bucket {owner}.{bucket} is not declared")]
50    BucketNotDeclared { owner: String, bucket: String },
51    #[error("module {module} may not read bucket {bucket}")]
52    BucketNotReadable { module: String, bucket: String },
53    #[error("module {module} does not own bucket {bucket}")]
54    BucketNotOwned { module: String, bucket: String },
55    #[error("bucket value is {size} bytes; the maximum is {cap} bytes")]
56    BucketValueTooLarge { size: usize, cap: u32 },
57    #[error("bucket {bucket} has no watch consumer slots available")]
58    WatchCapacity { bucket: String },
59}
60
61pub type Result<T> = std::result::Result<T, Error>;
62
63/// Options for one module's bus connection.
64#[derive(Clone, Debug)]
65#[non_exhaustive]
66pub struct BusClientOptions {
67    module: String,
68    pub request_timeout: Duration,
69}
70
71impl BusClientOptions {
72    pub fn new(module: impl Into<String>) -> Result<Self> {
73        let module = module.into();
74        validate_token(TokenKind::ModuleId, &module)?;
75        Ok(Self {
76            module,
77            request_timeout: Duration::from_secs(5),
78        })
79    }
80}
81
82/// A module's current connection and the task that maintains its credentials.
83pub struct BusClient {
84    connection: Arc<RwLock<NatsConnection>>,
85    names: AccountNames,
86    module: String,
87    changes: watch::Receiver<u64>,
88    task: tokio::task::JoinHandle<()>,
89}
90
91impl Drop for BusClient {
92    fn drop(&mut self) {
93        self.task.abort();
94    }
95}
96
97impl BusClient {
98    /// Issues a credential for the module identity that the subc daemon
99    /// authenticates from the module's launch nonce.
100    pub async fn connect(subc: Arc<SubcConsumer>, opts: BusClientOptions) -> Result<Self> {
101        let transport = SubcTransport::new(subc, &opts.module, opts.request_timeout);
102        // One module process can keep several connection snapshots, but issuing
103        // independently for them would revoke the other snapshots' credentials.
104        static MANAGERS: std::sync::LazyLock<
105            tokio::sync::Mutex<
106                std::collections::HashMap<String, std::sync::Weak<CredentialManager>>,
107            >,
108        > = std::sync::LazyLock::new(Default::default);
109        let mut managers = MANAGERS.lock().await;
110        let manager = match managers
111            .get(&opts.module)
112            .and_then(std::sync::Weak::upgrade)
113        {
114            Some(manager) => manager,
115            None => {
116                let manager = CredentialManager::start(Arc::new(transport)).await?;
117                managers.insert(opts.module.clone(), Arc::downgrade(&manager));
118                manager
119            }
120        };
121        drop(managers);
122        Self::connect_manager(manager, opts).await
123    }
124
125    async fn connect_manager(
126        manager: Arc<CredentialManager>,
127        opts: BusClientOptions,
128    ) -> Result<Self> {
129        let mut revisions = manager.revisions();
130        let initial = manager.credential().await;
131        let names = AccountNames::derive(&initial.acct)?;
132        let (connection, mut ended) = loop {
133            revisions.borrow_and_update();
134            let credential = manager.credential().await;
135            match connect_current(manager.clone(), &credential, opts.request_timeout).await {
136                Ok(current) if !revisions.has_changed().unwrap_or(false) => break current,
137                Ok((connection, _)) => {
138                    let _ = connection.close(Duration::from_millis(50)).await;
139                }
140                Err(_)
141                    if manager.credential().await.credential_public
142                        != credential.credential_public => {}
143                Err(error) => return Err(error),
144            }
145        };
146        let connection = Arc::new(RwLock::new(connection));
147        let shared = connection.clone();
148        let (changes, receiver) = watch::channel(0);
149        let task = tokio::spawn(async move {
150            let mut retry = None;
151            loop {
152                let credential = manager.credential().await;
153                let delay = retry
154                    .take()
155                    .unwrap_or_else(|| credentials::renewal_delay(credential.exp));
156                tokio::select! {
157                    result = revisions.changed() => { if result.is_err() { return; } }
158                    () = connection_ended(&mut ended) => {
159                        // Replace it through the same path as a re-issue, so
160                        // event readers and bucket watches rebind to the new one.
161                        tracing::warn!("bus connection closed for good; replacing it");
162                    }
163                    () = tokio::time::sleep(delay) => {
164                        if let Err(error) = manager.renew().await {
165                            if credentials::transient(&error) || matches!(error, Error::Transport(_)) {
166                                retry = Some(Duration::from_secs(5));
167                                continue;
168                            }
169                            // A terminal refusal (notably generation_not_live)
170                            // must not cause issuance under a dead generation.
171                            let refusal_code = match &error {
172                                Error::Refused(code) => code.as_str(),
173                                _ => "client_renewal_error",
174                            };
175                            tracing::warn!(
176                                refusal_code,
177                                "bus credential maintenance stopped; reconnect or report degraded"
178                            );
179                            return;
180                        }
181                        if !revisions.has_changed().unwrap_or(false) { continue; }
182                    }
183                }
184                let mut backoff = Duration::from_millis(250);
185                loop {
186                    let revision = *revisions.borrow_and_update();
187                    let credential = manager.credential().await;
188                    match connect_current(manager.clone(), &credential, opts.request_timeout).await
189                    {
190                        Ok((replacement, replacement_ended)) => {
191                            if revisions.has_changed().unwrap_or(false) {
192                                let _ = replacement.close(Duration::from_millis(50)).await;
193                                continue;
194                            }
195                            let old = {
196                                let mut slot = shared
197                                    .write()
198                                    .unwrap_or_else(std::sync::PoisonError::into_inner);
199                                std::mem::replace(&mut *slot, replacement)
200                            };
201                            ended = replacement_ended;
202                            changes.send_replace(revision);
203                            // A replaced connection that is reconnecting can't
204                            // drain, and its auth callback now refuses every
205                            // attempt; MAX_RECONNECTS ends it instead.
206                            let _ = old.close(Duration::from_millis(50)).await;
207                            break;
208                        }
209                        Err(_) => {
210                            tokio::time::sleep(backoff).await;
211                            backoff = (backoff * 2).min(Duration::from_secs(30));
212                        }
213                    }
214                }
215            }
216        });
217        Ok(Self {
218            connection,
219            names,
220            module: opts.module,
221            changes: receiver,
222            task,
223        })
224    }
225
226    /// Returns a snapshot. A re-issue does not change an already held snapshot.
227    pub fn connection(&self) -> NatsConnection {
228        self.connection
229            .read()
230            .unwrap_or_else(std::sync::PoisonError::into_inner)
231            .clone()
232    }
233
234    /// Changes only after a replacement connection with its issued inbox prefix is ready.
235    /// Reopen raw subscriptions and durable pulls when this receiver changes;
236    /// [`EventReader`] handles its own rebinding automatically.
237    ///
238    /// If credential maintenance can no longer renew or re-issue, its task ends
239    /// and drops the sender. The receiver's `changed()` then returns `Err` once
240    /// any unseen change has been consumed. Reconnect with [`BusClient::connect`]
241    /// or report the module as degraded when the change stream closes.
242    pub fn connection_changes(&self) -> watch::Receiver<u64> {
243        self.changes.clone()
244    }
245
246    pub fn acct(&self) -> &str {
247        self.names.account()
248    }
249
250    /// Returns the computer's box id as supplied by the bus module's account.
251    ///
252    /// Modules must never derive this id themselves. Returns `None` unless the
253    /// account is `box_` followed by a canonical 32-character box id.
254    pub fn box_id(&self) -> Option<cortexkit_bus_naming::federation::BoxId> {
255        credentials::box_id(self.acct())
256    }
257
258    /// Returns the inbox prefix used by the current connection.
259    ///
260    /// A per-credential prefix changes after credential re-issue; a module's
261    /// `ckdel` prefix stays stable. Use [`Self::connection_changes`] to observe
262    /// replacements, or [`NatsConnection::inbox_prefix`] for a held snapshot.
263    pub fn inbox_prefix(&self) -> String {
264        self.connection().inbox_prefix().to_owned()
265    }
266
267    /// Attaches to an existing EVENT durable owned by this module.
268    ///
269    /// ck-bus provisions durables; this method never creates or changes one.
270    /// The name must follow bus-naming's `m_{module}[_suffix]` owner rule.
271    /// The reader rebinds automatically after credential re-issue. Nothing is
272    /// acknowledged automatically: commit your own transaction before `ack()`.
273    pub async fn event_reader(&self, durable: &str) -> Result<EventReader> {
274        EventReader::attach(self, durable).await
275    }
276
277    /// Opens a declared module bucket for its owner or a declared reader.
278    /// Does not provision buckets or change their limits.
279    pub fn bucket(&self, owner_module: &str, bucket: &str) -> Result<Bucket> {
280        Bucket::attach(self, owner_module, bucket)
281    }
282
283    /// Stores an event using the fact's own id for JetStream deduplication.
284    ///
285    /// Event bodies carry ids, counters and enums, not large content. Bodies
286    /// larger than 4 KiB are refused; retrieve large content through its owner's
287    /// own call instead.
288    /// A non-empty body is sent with its SHA-256 as `Ck-Content-Digest`, so a
289    /// reader can check the bytes it received. An empty body is sent without one,
290    /// because an empty payload with a digest is the claim-check shape.
291    /// Inline bodies reach every reader whose durable matches the subject, with
292    /// no per-reader visibility check; reserve them for pointers and facts every
293    /// declared reader may see. Flow triggers must use [`Self::publish_claim_check`],
294    /// the `cortexkit_bus_trait::Stream::publish` claim-check path (empty payload plus
295    /// content digest), with the publisher serving the body under the reading
296    /// flow's own scope to check which agent may see it. The flow engine refuses
297    /// inline trigger bodies.
298    pub async fn publish_event(
299        &self,
300        event: &str,
301        version: u32,
302        qualifier: Option<&str>,
303        msg_id: &str,
304        body: impl Into<Bytes>,
305    ) -> Result<PublishReceipt> {
306        let body = body.into();
307        if body.len() > 4096 {
308            return Err(Error::EventBodyTooLarge { size: body.len() });
309        }
310        let subject = self.event_subject(event, version, qualifier)?;
311        let mut headers = async_nats::HeaderMap::new();
312        headers.insert(async_nats::header::NATS_MESSAGE_ID, msg_id);
313        if !body.is_empty() {
314            headers.insert(
315                CONTENT_DIGEST_HEADER,
316                ContentDigest::of_bytes(&body).to_string().as_str(),
317            );
318        }
319        let ack = async_nats::jetstream::new(self.connection().client())
320            .publish_with_headers(subject, headers, body)
321            .await
322            .map_err(|error| Error::Nats(error.to_string()))?
323            .await
324            .map_err(|error| Error::Nats(error.to_string()))?;
325        Ok(PublishReceipt::new(ack.sequence, ack.duplicate))
326    }
327
328    /// Publishes an empty-payload claim check carrying the body's content digest.
329    /// Store the body before publishing and serve it through the publisher's own
330    /// `events_get` query under the reading flow's scope. This is the event shape
331    /// for triggering flows; inline bodies cannot be used as flow triggers.
332    /// The NATS codec writes Nats-Msg-Id and Ck-Content-Digest and refuses caller
333    /// headers spelling either reserved name, including case variants.
334    #[allow(clippy::too_many_arguments)]
335    pub async fn publish_claim_check(
336        &self,
337        event: &str,
338        version: u32,
339        qualifier: Option<&str>,
340        msg_id: &str,
341        digest: ContentDigest,
342        headers: Headers,
343    ) -> Result<PublishReceipt> {
344        let subject = self.event_subject(event, version, qualifier)?;
345        self.connection()
346            .stream(&self.names.streams().event)
347            .publish(&subject, msg_id, digest, headers)
348            .await
349            .map_err(Error::from)
350    }
351
352    fn event_subject(&self, event: &str, version: u32, qualifier: Option<&str>) -> Result<String> {
353        Ok(match qualifier {
354            Some(reader) => {
355                self.names
356                    .event_subject_with_reader(&self.module, event, version, reader)?
357            }
358            None => self.names.event_subject(&self.module, event, version)?,
359        })
360    }
361
362    /// Publishes an ephemeral, versioned hint, not a stored fact.
363    pub async fn live_hint(&self, kind: &str, version: u32, body: impl Into<Bytes>) -> Result<()> {
364        let subject = self.names.live_subject(&self.module, kind, version)?;
365        self.connection()
366            .client()
367            .publish(subject, body.into())
368            .await
369            .map_err(|error| Error::Nats(error.to_string()))
370    }
371}
372
373/// How many reconnect attempts async-nats makes after a connection drops before
374/// it gives up and ends the connection for good.
375///
376/// async-nats waits nothing before the first attempt, then 2 ms doubling up to
377/// a 4 s cap, so twenty attempts span about 36 s: a ck-bus restart of about
378/// 30 s rides through on the same connection. The bound matters most for a
379/// replaced connection. Its credential is no longer current, so its auth
380/// callback refuses every attempt; without a bound it would keep connecting
381/// and failing every 4 s for the life of the process.
382const MAX_RECONNECTS: usize = 20;
383
384/// A subject under the connection's own inbox that nothing ever publishes to.
385/// A hyphen never appears in async-nats's generated inbox tokens, so it
386/// cannot collide with a request inbox.
387pub(crate) const LIVENESS_SUBJECT: &str = "bus-client-liveness";
388
389/// Resolves once async-nats has ended the connection for good: after it runs
390/// out of reconnect attempts, or after a drain. The connection's handler task
391/// then exits and drops every subscription, which ends this stream. The
392/// subscription survives ordinary reconnects because async-nats resubscribes it.
393/// async-nats 0.50 emits no `Event::Closed` when it gives up reconnecting, only
394/// a `ClientError::MaxReconnects` event that can be dropped from a full queue,
395/// and `NatsConnection` keeps its event stream private; the ended subscription
396/// is the signal that can't be missed.
397async fn connection_ended(ended: &mut async_nats::Subscriber) {
398    use futures_util::StreamExt;
399    while ended.next().await.is_some() {}
400}
401
402async fn connect_current(
403    manager: Arc<CredentialManager>,
404    credential: &credentials::Credential,
405    timeout: Duration,
406) -> Result<(NatsConnection, async_nats::Subscriber)> {
407    let public = credential.credential_public.clone();
408    let options = async_nats::ConnectOptions::with_auth_callback(move |nonce| {
409        let manager = manager.clone();
410        let public = public.clone();
411        // The spawned subc request future only needs to be Send; async-nats
412        // also requires the callback future to be Sync.
413        let task = tokio::spawn(async move {
414            let credential = manager.credential().await;
415            if credential.credential_public != public {
416                return Err(Error::Reply("connection credential was replaced"));
417            }
418            let signature = manager.sign(&public, nonce).await?;
419            let mut auth = async_nats::Auth::new();
420            auth.jwt = Some(credential.jwt);
421            auth.signature = Some(signature);
422            Ok(auth)
423        });
424        async move {
425            task.await
426                .map_err(async_nats::AuthError::new)?
427                .map_err(async_nats::AuthError::new)
428        }
429    })
430    .connection_timeout(timeout)
431    .request_timeout(Some(timeout))
432    .max_reconnects(MAX_RECONNECTS);
433    let mut config = ConnectConfig::new(&credential.credential_public)?;
434    config.request_timeout = timeout;
435    let connection = if credential.inbox_prefix == config.inbox_prefix() {
436        NatsConnection::connect(credential.server_url.as_str(), options, config).await
437    } else {
438        NatsConnection::connect_with_inbox_prefix(
439            credential.server_url.as_str(),
440            options,
441            config,
442            &credential.inbox_prefix,
443        )
444        .await
445    }
446    .map_err(Error::from)?;
447    let subject = format!("{}.{LIVENESS_SUBJECT}", connection.inbox_prefix());
448    match connection.client().subscribe(subject).await {
449        Ok(ended) => Ok((connection, ended)),
450        Err(error) => {
451            let _ = connection.close(Duration::from_millis(50)).await;
452            Err(Error::Nats(error.to_string()))
453        }
454    }
455}
456
457#[cfg(test)]
458mod tests;