cortexkit-bus-client 0.1.6

Seedless ck-bus credentials and local NATS connections for Rust modules
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
#![doc = include_str!("../README.md")]

mod binding;
mod buckets;
mod credentials;
mod events;

pub use buckets::{Bucket, BucketEntry, BucketItem, BucketWatch};
pub use events::{EventDelivery, EventItem, EventReader, MalformedDelivery, MalformedReason};

use std::{
    sync::{Arc, RwLock},
    time::Duration,
};

use bytes::Bytes;
use cortexkit_bus_naming::{validate_token, AccountNames, TokenKind};
use cortexkit_bus_nats::{ConnectConfig, NatsConnection};
use cortexkit_bus_trait::{ContentDigest, Headers, PublishReceipt, Stream};

/// The header carrying a body's SHA-256, spelled as the NATS codec writes it
/// for claim checks.
const CONTENT_DIGEST_HEADER: &str = "Ck-Content-Digest";
use subc_client_rs::SubcConsumer;
use tokio::sync::watch;

use credentials::{CredentialManager, SubcTransport};

/// Errors contain refusal codes, not credential JWTs or nonce signatures.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
    #[error("ck-bus refused the request: {0}")]
    Refused(String),
    #[error("ck-bus request failed: {0}")]
    Transport(String),
    #[error("invalid ck-bus reply: {0}")]
    Reply(&'static str),
    #[error(transparent)]
    Naming(#[from] cortexkit_bus_naming::NamingError),
    #[error(transparent)]
    Bus(#[from] cortexkit_bus_trait::BusError),
    #[error("NATS operation failed: {0}")]
    Nats(String),
    #[error("event body is {size} bytes; the maximum is 4096 bytes")]
    EventBodyTooLarge { size: usize },
    #[error("durable {durable} is not owned by module {module}")]
    DurableNotOwned { durable: String, module: String },
    #[error("module bucket {owner}.{bucket} is not declared")]
    BucketNotDeclared { owner: String, bucket: String },
    #[error("module {module} may not read bucket {bucket}")]
    BucketNotReadable { module: String, bucket: String },
    #[error("module {module} does not own bucket {bucket}")]
    BucketNotOwned { module: String, bucket: String },
    #[error("bucket value is {size} bytes; the maximum is {cap} bytes")]
    BucketValueTooLarge { size: usize, cap: u32 },
    #[error("bucket {bucket} has no watch consumer slots available")]
    WatchCapacity { bucket: String },
}

pub type Result<T> = std::result::Result<T, Error>;

/// Options for one module's bus connection.
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct BusClientOptions {
    module: String,
    pub request_timeout: Duration,
}

impl BusClientOptions {
    pub fn new(module: impl Into<String>) -> Result<Self> {
        let module = module.into();
        validate_token(TokenKind::ModuleId, &module)?;
        Ok(Self {
            module,
            request_timeout: Duration::from_secs(5),
        })
    }
}

/// A module's current connection and the task that maintains its credentials.
pub struct BusClient {
    connection: Arc<RwLock<NatsConnection>>,
    names: AccountNames,
    module: String,
    changes: watch::Receiver<u64>,
    task: tokio::task::JoinHandle<()>,
}

impl Drop for BusClient {
    fn drop(&mut self) {
        self.task.abort();
    }
}

impl BusClient {
    /// Issues a credential for the module identity that the subc daemon
    /// authenticates from the module's launch nonce.
    pub async fn connect(subc: Arc<SubcConsumer>, opts: BusClientOptions) -> Result<Self> {
        let transport = SubcTransport::new(subc, &opts.module, opts.request_timeout);
        // One module process can keep several connection snapshots, but issuing
        // independently for them would revoke the other snapshots' credentials.
        static MANAGERS: std::sync::LazyLock<
            tokio::sync::Mutex<
                std::collections::HashMap<String, std::sync::Weak<CredentialManager>>,
            >,
        > = std::sync::LazyLock::new(Default::default);
        let mut managers = MANAGERS.lock().await;
        let manager = match managers
            .get(&opts.module)
            .and_then(std::sync::Weak::upgrade)
        {
            Some(manager) => manager,
            None => {
                let manager = CredentialManager::start(Arc::new(transport)).await?;
                managers.insert(opts.module.clone(), Arc::downgrade(&manager));
                manager
            }
        };
        drop(managers);
        Self::connect_manager(manager, opts).await
    }

    async fn connect_manager(
        manager: Arc<CredentialManager>,
        opts: BusClientOptions,
    ) -> Result<Self> {
        let mut revisions = manager.revisions();
        let initial = manager.credential().await;
        let names = AccountNames::derive(&initial.acct)?;
        let (connection, mut ended) = loop {
            revisions.borrow_and_update();
            let credential = manager.credential().await;
            match connect_current(manager.clone(), &credential, opts.request_timeout).await {
                Ok(current) if !revisions.has_changed().unwrap_or(false) => break current,
                Ok((connection, _)) => {
                    let _ = connection.close(Duration::from_millis(50)).await;
                }
                Err(_)
                    if manager.credential().await.credential_public
                        != credential.credential_public => {}
                Err(error) => return Err(error),
            }
        };
        let connection = Arc::new(RwLock::new(connection));
        let shared = connection.clone();
        let (changes, receiver) = watch::channel(0);
        let task = tokio::spawn(async move {
            let mut retry = None;
            loop {
                let credential = manager.credential().await;
                let delay = retry
                    .take()
                    .unwrap_or_else(|| credentials::renewal_delay(credential.exp));
                tokio::select! {
                    result = revisions.changed() => { if result.is_err() { return; } }
                    () = connection_ended(&mut ended) => {
                        // Replace it through the same path as a re-issue, so
                        // event readers and bucket watches rebind to the new one.
                        tracing::warn!("bus connection closed for good; replacing it");
                    }
                    () = tokio::time::sleep(delay) => {
                        if let Err(error) = manager.renew().await {
                            if credentials::transient(&error) || matches!(error, Error::Transport(_)) {
                                retry = Some(Duration::from_secs(5));
                                continue;
                            }
                            // A terminal refusal (notably generation_not_live)
                            // must not cause issuance under a dead generation.
                            let refusal_code = match &error {
                                Error::Refused(code) => code.as_str(),
                                _ => "client_renewal_error",
                            };
                            tracing::warn!(
                                refusal_code,
                                "bus credential maintenance stopped; reconnect or report degraded"
                            );
                            return;
                        }
                        if !revisions.has_changed().unwrap_or(false) { continue; }
                    }
                }
                let mut backoff = Duration::from_millis(250);
                loop {
                    let revision = *revisions.borrow_and_update();
                    let credential = manager.credential().await;
                    match connect_current(manager.clone(), &credential, opts.request_timeout).await
                    {
                        Ok((replacement, replacement_ended)) => {
                            if revisions.has_changed().unwrap_or(false) {
                                let _ = replacement.close(Duration::from_millis(50)).await;
                                continue;
                            }
                            let old = {
                                let mut slot = shared
                                    .write()
                                    .unwrap_or_else(std::sync::PoisonError::into_inner);
                                std::mem::replace(&mut *slot, replacement)
                            };
                            ended = replacement_ended;
                            changes.send_replace(revision);
                            // A replaced connection that is reconnecting can't
                            // drain, and its auth callback now refuses every
                            // attempt; MAX_RECONNECTS ends it instead.
                            let _ = old.close(Duration::from_millis(50)).await;
                            break;
                        }
                        Err(_) => {
                            tokio::time::sleep(backoff).await;
                            backoff = (backoff * 2).min(Duration::from_secs(30));
                        }
                    }
                }
            }
        });
        Ok(Self {
            connection,
            names,
            module: opts.module,
            changes: receiver,
            task,
        })
    }

    /// Returns a snapshot. A re-issue does not change an already held snapshot.
    pub fn connection(&self) -> NatsConnection {
        self.connection
            .read()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
            .clone()
    }

    /// Changes only after a replacement connection with a new inbox is ready.
    /// Reopen raw subscriptions and durable pulls when this receiver changes;
    /// [`EventReader`] handles its own rebinding automatically.
    ///
    /// If credential maintenance can no longer renew or re-issue, its task ends
    /// and drops the sender. The receiver's `changed()` then returns `Err` once
    /// any unseen change has been consumed. Reconnect with [`BusClient::connect`]
    /// or report the module as degraded when the change stream closes.
    pub fn connection_changes(&self) -> watch::Receiver<u64> {
        self.changes.clone()
    }

    pub fn acct(&self) -> &str {
        self.names.account()
    }

    /// Attaches to an existing EVENT durable owned by this module.
    ///
    /// ck-bus provisions durables; this method never creates or changes one.
    /// The name must follow bus-naming's `m_{module}[_suffix]` owner rule.
    /// The reader rebinds automatically after credential re-issue. Nothing is
    /// acknowledged automatically: commit your own transaction before `ack()`.
    pub async fn event_reader(&self, durable: &str) -> Result<EventReader> {
        EventReader::attach(self, durable).await
    }

    /// Opens a declared module bucket for its owner or a declared reader.
    /// Does not provision buckets or change their limits.
    pub fn bucket(&self, owner_module: &str, bucket: &str) -> Result<Bucket> {
        Bucket::attach(self, owner_module, bucket)
    }

    /// Stores an event using the fact's own id for JetStream deduplication.
    ///
    /// Event bodies carry ids, counters and enums, not large content. Bodies
    /// larger than 4 KiB are refused; retrieve large content through its owner's
    /// own call instead.
    /// A non-empty body is sent with its SHA-256 as `Ck-Content-Digest`, so a
    /// reader can check the bytes it received. An empty body is sent without one,
    /// because an empty payload with a digest is the claim-check shape.
    /// Inline bodies reach every reader whose durable matches the subject, with
    /// no per-reader visibility check; reserve them for pointers and facts every
    /// declared reader may see. Flow triggers must use [`Self::publish_claim_check`],
    /// the `cortexkit_bus_trait::Stream::publish` claim-check path (empty payload plus
    /// content digest), with the publisher serving the body under the reading
    /// flow's own scope to check which agent may see it. The flow engine refuses
    /// inline trigger bodies.
    pub async fn publish_event(
        &self,
        event: &str,
        version: u32,
        qualifier: Option<&str>,
        msg_id: &str,
        body: impl Into<Bytes>,
    ) -> Result<PublishReceipt> {
        let body = body.into();
        if body.len() > 4096 {
            return Err(Error::EventBodyTooLarge { size: body.len() });
        }
        let subject = self.event_subject(event, version, qualifier)?;
        let mut headers = async_nats::HeaderMap::new();
        headers.insert(async_nats::header::NATS_MESSAGE_ID, msg_id);
        if !body.is_empty() {
            headers.insert(
                CONTENT_DIGEST_HEADER,
                ContentDigest::of_bytes(&body).to_string().as_str(),
            );
        }
        let ack = async_nats::jetstream::new(self.connection().client())
            .publish_with_headers(subject, headers, body)
            .await
            .map_err(|error| Error::Nats(error.to_string()))?
            .await
            .map_err(|error| Error::Nats(error.to_string()))?;
        Ok(PublishReceipt::new(ack.sequence, ack.duplicate))
    }

    /// Publishes an empty-payload claim check carrying the body's content digest.
    /// Store the body before publishing and serve it through the publisher's own
    /// `events_get` query under the reading flow's scope. This is the event shape
    /// for triggering flows; inline bodies cannot be used as flow triggers.
    /// The NATS codec writes Nats-Msg-Id and Ck-Content-Digest and refuses caller
    /// headers spelling either reserved name, including case variants.
    #[allow(clippy::too_many_arguments)]
    pub async fn publish_claim_check(
        &self,
        event: &str,
        version: u32,
        qualifier: Option<&str>,
        msg_id: &str,
        digest: ContentDigest,
        headers: Headers,
    ) -> Result<PublishReceipt> {
        let subject = self.event_subject(event, version, qualifier)?;
        self.connection()
            .stream(&self.names.streams().event)
            .publish(&subject, msg_id, digest, headers)
            .await
            .map_err(Error::from)
    }

    fn event_subject(&self, event: &str, version: u32, qualifier: Option<&str>) -> Result<String> {
        Ok(match qualifier {
            Some(reader) => {
                self.names
                    .event_subject_with_reader(&self.module, event, version, reader)?
            }
            None => self.names.event_subject(&self.module, event, version)?,
        })
    }

    /// Publishes an ephemeral, versioned hint, not a stored fact.
    pub async fn live_hint(&self, kind: &str, version: u32, body: impl Into<Bytes>) -> Result<()> {
        let subject = self.names.live_subject(&self.module, kind, version)?;
        self.connection()
            .client()
            .publish(subject, body.into())
            .await
            .map_err(|error| Error::Nats(error.to_string()))
    }
}

/// How many reconnect attempts async-nats makes after a connection drops before
/// it gives up and ends the connection for good.
///
/// async-nats waits nothing before the first attempt, then 2 ms doubling up to
/// a 4 s cap, so twenty attempts span about 36 s: a ck-bus restart of about
/// 30 s rides through on the same connection. The bound matters most for a
/// replaced connection. Its credential is no longer current, so its auth
/// callback refuses every attempt; without a bound it would keep connecting
/// and failing every 4 s for the life of the process.
const MAX_RECONNECTS: usize = 20;

/// A subject under the connection's own inbox that nothing ever publishes to.
/// A hyphen never appears in async-nats's generated inbox tokens, so it
/// cannot collide with a request inbox.
pub(crate) const LIVENESS_SUBJECT: &str = "bus-client-liveness";

/// Resolves once async-nats has ended the connection for good: after it runs
/// out of reconnect attempts, or after a drain. The connection's handler task
/// then exits and drops every subscription, which ends this stream. The
/// subscription survives ordinary reconnects because async-nats resubscribes it.
/// async-nats 0.50 emits no `Event::Closed` when it gives up reconnecting, only
/// a `ClientError::MaxReconnects` event that can be dropped from a full queue,
/// and `NatsConnection` keeps its event stream private; the ended subscription
/// is the signal that can't be missed.
async fn connection_ended(ended: &mut async_nats::Subscriber) {
    use futures_util::StreamExt;
    while ended.next().await.is_some() {}
}

async fn connect_current(
    manager: Arc<CredentialManager>,
    credential: &credentials::Credential,
    timeout: Duration,
) -> Result<(NatsConnection, async_nats::Subscriber)> {
    let public = credential.credential_public.clone();
    let options = async_nats::ConnectOptions::with_auth_callback(move |nonce| {
        let manager = manager.clone();
        let public = public.clone();
        // The spawned subc request future only needs to be Send; async-nats
        // also requires the callback future to be Sync.
        let task = tokio::spawn(async move {
            let credential = manager.credential().await;
            if credential.credential_public != public {
                return Err(Error::Reply("connection credential was replaced"));
            }
            let signature = manager.sign(&public, nonce).await?;
            let mut auth = async_nats::Auth::new();
            auth.jwt = Some(credential.jwt);
            auth.signature = Some(signature);
            Ok(auth)
        });
        async move {
            task.await
                .map_err(async_nats::AuthError::new)?
                .map_err(async_nats::AuthError::new)
        }
    })
    .connection_timeout(timeout)
    .request_timeout(Some(timeout))
    .max_reconnects(MAX_RECONNECTS);
    let mut config = ConnectConfig::new(&credential.credential_public)?;
    config.request_timeout = timeout;
    let connection = NatsConnection::connect(credential.server_url.as_str(), options, config)
        .await
        .map_err(Error::from)?;
    let subject = format!("{}.{LIVENESS_SUBJECT}", connection.inbox_prefix());
    match connection.client().subscribe(subject).await {
        Ok(ended) => Ok((connection, ended)),
        Err(error) => {
            let _ = connection.close(Duration::from_millis(50)).await;
            Err(Error::Nats(error.to_string()))
        }
    }
}

#[cfg(test)]
mod tests;