Skip to main content

nidus_kafka/
lib.rs

1#![forbid(unsafe_code)]
2#![deny(missing_docs)]
3
4//! First-party Apache Kafka integration for Nidus.
5//!
6//! The adapter preserves rust-rdkafka's native producer, consumer, admin,
7//! partition, offset, rebalance, and transaction APIs. Convenience publishing
8//! waits for Kafka delivery reports and defaults to idempotent production, but
9//! Nidus does not claim end-to-end exactly-once processing.
10
11use std::{
12    collections::BTreeMap,
13    fmt,
14    sync::{
15        Arc,
16        atomic::{AtomicBool, Ordering},
17    },
18    time::{Duration, Instant},
19};
20
21use async_trait::async_trait;
22use nidus_core::{Container, LifecycleHook, NidusError};
23use nidus_integrations::{
24    IntegrationEvent, IntegrationStatus, IntegrationTelemetry, MessageEnvelope,
25};
26use rdkafka::{
27    ClientConfig, Message,
28    admin::AdminClient,
29    client::DefaultClientContext,
30    consumer::StreamConsumer,
31    error::KafkaError as NativeKafkaError,
32    message::{Header, OwnedHeaders},
33    producer::{FutureProducer, FutureRecord, Producer},
34};
35use serde::Serialize;
36use thiserror::Error;
37
38/// Result type for Kafka adapter operations.
39pub type Result<T> = std::result::Result<T, KafkaError>;
40
41/// Kafka adapter error preserving native rust-rdkafka errors.
42#[derive(Debug, Error)]
43pub enum KafkaError {
44    /// rust-rdkafka returned an error.
45    #[error(transparent)]
46    Kafka(#[from] NativeKafkaError),
47    /// A delivery report failed after the message entered the producer queue.
48    #[error("Kafka delivery failed: {source}")]
49    Delivery {
50        /// Native delivery failure.
51        #[source]
52        source: NativeKafkaError,
53    },
54    /// A blocking librdkafka operation could not be joined.
55    #[error("Kafka blocking task failed")]
56    TaskJoin,
57    /// Shared envelope serialization failed.
58    #[error(transparent)]
59    Integration(#[from] nidus_integrations::IntegrationError),
60    /// Nidus provider registration failed.
61    #[error(transparent)]
62    Nidus(#[from] NidusError),
63    /// Adapter configuration was unsafe or incomplete.
64    #[error("invalid Kafka configuration: {message}")]
65    Configuration {
66        /// Redaction-safe validation message.
67        message: &'static str,
68    },
69    /// The provider stopped accepting adapter-owned publications.
70    #[error("Kafka provider is shutting down")]
71    ShuttingDown,
72}
73
74/// Typed Kafka client configuration with secure production defaults.
75#[derive(Clone, Eq, PartialEq)]
76pub struct KafkaConfig {
77    bootstrap_servers: String,
78    client_id: String,
79    properties: BTreeMap<String, String>,
80    enqueue_timeout: Duration,
81    shutdown_timeout: Duration,
82    allow_plaintext_local: bool,
83}
84
85impl KafkaConfig {
86    /// Creates Kafka configuration using TLS, idempotent production, and bounded queues.
87    pub fn new(bootstrap_servers: impl Into<String>, client_id: impl Into<String>) -> Self {
88        Self {
89            bootstrap_servers: bootstrap_servers.into(),
90            client_id: client_id.into(),
91            properties: BTreeMap::from([
92                ("security.protocol".to_owned(), "SSL".to_owned()),
93                ("enable.idempotence".to_owned(), "true".to_owned()),
94                ("acks".to_owned(), "all".to_owned()),
95                ("message.timeout.ms".to_owned(), "30000".to_owned()),
96                (
97                    "queue.buffering.max.messages".to_owned(),
98                    "100000".to_owned(),
99                ),
100            ]),
101            enqueue_timeout: Duration::from_secs(5),
102            shutdown_timeout: Duration::from_secs(10),
103            allow_plaintext_local: false,
104        }
105    }
106
107    /// Adds or replaces a native librdkafka property.
108    pub fn property(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
109        self.properties.insert(key.into(), value.into());
110        self
111    }
112
113    /// Explicitly permits a plaintext broker for local development and tests.
114    pub fn allow_plaintext_for_local_development(mut self) -> Self {
115        self.properties
116            .insert("security.protocol".to_owned(), "PLAINTEXT".to_owned());
117        self.allow_plaintext_local = true;
118        self
119    }
120
121    /// Sets how long a producer call may wait for local queue capacity.
122    pub fn with_enqueue_timeout(mut self, value: Duration) -> Self {
123        self.enqueue_timeout = value;
124        self
125    }
126
127    /// Sets the graceful producer flush timeout.
128    pub fn with_shutdown_timeout(mut self, value: Duration) -> Self {
129        self.shutdown_timeout = value;
130        self
131    }
132
133    /// Returns the bootstrap server list.
134    pub fn bootstrap_servers(&self) -> &str {
135        &self.bootstrap_servers
136    }
137
138    /// Returns the stable client id.
139    pub fn client_id(&self) -> &str {
140        &self.client_id
141    }
142
143    /// Returns configured native property names and values.
144    pub fn properties(&self) -> &BTreeMap<String, String> {
145        &self.properties
146    }
147
148    /// Validates safe, bounded client settings before native client creation.
149    pub fn validate(&self) -> Result<()> {
150        if self.bootstrap_servers.trim().is_empty() {
151            return Err(KafkaError::Configuration {
152                message: "bootstrap_servers cannot be empty",
153            });
154        }
155        if self.client_id.trim().is_empty() {
156            return Err(KafkaError::Configuration {
157                message: "client_id cannot be empty",
158            });
159        }
160        if self.enqueue_timeout.is_zero() || self.shutdown_timeout.is_zero() {
161            return Err(KafkaError::Configuration {
162                message: "Kafka timeouts must be greater than zero",
163            });
164        }
165        let protocol = self
166            .properties
167            .get("security.protocol")
168            .map(String::as_str)
169            .unwrap_or("PLAINTEXT");
170        let plaintext = protocol.eq_ignore_ascii_case("PLAINTEXT")
171            || protocol.eq_ignore_ascii_case("SASL_PLAINTEXT");
172        if plaintext && !self.allow_plaintext_local {
173            return Err(KafkaError::Configuration {
174                message: "plaintext Kafka requires an explicit local-development opt-in",
175            });
176        }
177        if plaintext && !kafka_bootstrap_is_loopback(&self.bootstrap_servers) {
178            return Err(KafkaError::Configuration {
179                message: "plaintext Kafka is restricted to loopback brokers",
180            });
181        }
182        Ok(())
183    }
184
185    fn native_config(&self) -> Result<ClientConfig> {
186        self.validate()?;
187        let mut config = ClientConfig::new();
188        config
189            .set("bootstrap.servers", &self.bootstrap_servers)
190            .set("client.id", &self.client_id);
191        for (key, value) in &self.properties {
192            config.set(key, value);
193        }
194        Ok(config)
195    }
196}
197
198fn kafka_bootstrap_is_loopback(bootstrap_servers: &str) -> bool {
199    bootstrap_servers
200        .split(',')
201        .all(|server| kafka_broker_is_loopback(server.trim()))
202}
203
204fn kafka_broker_is_loopback(server: &str) -> bool {
205    if server.is_empty() || server.contains('@') {
206        return false;
207    }
208    let Some((host, port)) = server.rsplit_once(':') else {
209        return false;
210    };
211    if port.parse::<u16>().is_err() {
212        return false;
213    }
214    let host = host
215        .strip_prefix('[')
216        .and_then(|host| host.strip_suffix(']'))
217        .unwrap_or(host);
218    host.eq_ignore_ascii_case("localhost")
219        || host
220            .parse::<std::net::IpAddr>()
221            .is_ok_and(|address| address.is_loopback())
222}
223
224impl fmt::Debug for KafkaConfig {
225    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
226        formatter
227            .debug_struct("KafkaConfig")
228            .field("bootstrap_servers", &"<redacted>")
229            .field("client_id", &self.client_id)
230            .field(
231                "property_names",
232                &self.properties.keys().collect::<Vec<_>>(),
233            )
234            .field("enqueue_timeout", &self.enqueue_timeout)
235            .field("shutdown_timeout", &self.shutdown_timeout)
236            .field("allow_plaintext_local", &self.allow_plaintext_local)
237            .finish()
238    }
239}
240
241/// Successful Kafka delivery coordinates.
242#[derive(Clone, Copy, Debug, Eq, PartialEq)]
243pub struct KafkaDelivery {
244    partition: i32,
245    offset: i64,
246}
247
248impl KafkaDelivery {
249    /// Returns the destination partition.
250    pub const fn partition(&self) -> i32 {
251        self.partition
252    }
253
254    /// Returns the committed log offset from the delivery report.
255    pub const fn offset(&self) -> i64 {
256        self.offset
257    }
258}
259
260/// Builder for a Kafka provider.
261#[derive(Clone, Debug)]
262pub struct KafkaProviderBuilder {
263    config: KafkaConfig,
264    telemetry: IntegrationTelemetry,
265}
266
267impl KafkaProviderBuilder {
268    /// Creates a provider builder.
269    pub fn new(config: KafkaConfig) -> Self {
270        Self {
271            config,
272            telemetry: IntegrationTelemetry::new(),
273        }
274    }
275
276    /// Adds shared tracing, metrics, dashboard, or custom telemetry.
277    pub fn telemetry(mut self, telemetry: IntegrationTelemetry) -> Self {
278        self.telemetry = telemetry;
279        self
280    }
281
282    /// Creates native Kafka clients without blocking on broker connectivity.
283    pub fn build(self) -> Result<KafkaProvider> {
284        let native = self.config.native_config()?;
285        let producer = native.create::<FutureProducer>()?;
286        let admin = native.create::<AdminClient<DefaultClientContext>>()?;
287        Ok(KafkaProvider {
288            config: self.config,
289            producer,
290            admin: Arc::new(admin),
291            shutting_down: Arc::new(AtomicBool::new(false)),
292            telemetry: self.telemetry,
293        })
294    }
295
296    /// Builds and registers the provider as a Nidus singleton.
297    pub fn register(self, container: &mut Container) -> Result<()> {
298        container.register_singleton(self.build()?)?;
299        Ok(())
300    }
301}
302
303/// Nidus provider exposing native rust-rdkafka clients.
304#[derive(Clone)]
305pub struct KafkaProvider {
306    config: KafkaConfig,
307    producer: FutureProducer,
308    admin: Arc<AdminClient<DefaultClientContext>>,
309    shutting_down: Arc<AtomicBool>,
310    telemetry: IntegrationTelemetry,
311}
312
313impl KafkaProvider {
314    /// Creates a Kafka provider builder.
315    pub fn builder(config: KafkaConfig) -> KafkaProviderBuilder {
316        KafkaProviderBuilder::new(config)
317    }
318
319    /// Returns the native asynchronous producer.
320    pub fn producer(&self) -> &FutureProducer {
321        &self.producer
322    }
323
324    /// Returns the native Kafka admin client.
325    pub fn admin(&self) -> &AdminClient<DefaultClientContext> {
326        &self.admin
327    }
328
329    /// Creates a native manual-commit stream consumer for a group.
330    ///
331    /// Auto commit and auto offset storage are disabled so applications can
332    /// store or commit offsets only after successful processing.
333    pub fn consumer(&self, group_id: &str) -> Result<StreamConsumer> {
334        if group_id.trim().is_empty() {
335            return Err(KafkaError::Configuration {
336                message: "Kafka consumer group_id cannot be empty",
337            });
338        }
339        let mut config = self.config.native_config()?;
340        config
341            .set("group.id", group_id)
342            .set("enable.auto.commit", "false")
343            .set("enable.auto.offset.store", "false")
344            .set("isolation.level", "read_committed");
345        Ok(config.create()?)
346    }
347
348    /// Publishes bytes and waits for the native Kafka delivery report.
349    pub async fn publish(
350        &self,
351        topic: &str,
352        key: Option<&[u8]>,
353        payload: &[u8],
354    ) -> Result<KafkaDelivery> {
355        self.publish_with_headers(topic, key, payload, None).await
356    }
357
358    /// Publishes a shared envelope with correlation and trace headers.
359    pub async fn publish_envelope<T>(
360        &self,
361        topic: &str,
362        key: Option<&[u8]>,
363        envelope: &MessageEnvelope<T>,
364    ) -> Result<KafkaDelivery>
365    where
366        T: Serialize,
367    {
368        let payload = envelope.to_json()?;
369        let mut headers = OwnedHeaders::new()
370            .insert(Header {
371                key: "nidus-message-id",
372                value: Some(envelope.id()),
373            })
374            .insert(Header {
375                key: "nidus-message-name",
376                value: Some(envelope.name()),
377            });
378        if let Some(value) = envelope.metadata().correlation_id_value() {
379            headers = headers.insert(Header {
380                key: "nidus-correlation-id",
381                value: Some(value),
382            });
383        }
384        if let Some(value) = envelope.metadata().traceparent_value() {
385            headers = headers.insert(Header {
386                key: "traceparent",
387                value: Some(value),
388            });
389        }
390        self.publish_with_headers(topic, key, &payload, Some(headers))
391            .await
392    }
393
394    /// Flushes queued delivery reports without blocking a Tokio worker thread.
395    pub async fn shutdown(&self) -> Result<()> {
396        self.shutting_down.store(true, Ordering::Release);
397        let producer = self.producer.clone();
398        let timeout = self.config.shutdown_timeout;
399        tokio::task::spawn_blocking(move || producer.flush(timeout))
400            .await
401            .map_err(|_| KafkaError::TaskJoin)??;
402        Ok(())
403    }
404
405    /// Checks broker metadata without blocking a Tokio worker thread.
406    #[cfg(feature = "health")]
407    pub async fn health_status(&self) -> nidus_http::health::HealthStatus {
408        if self.shutting_down.load(Ordering::Acquire) {
409            return nidus_http::health::HealthStatus::down("kafka provider is shutting down");
410        }
411        let started_at = Instant::now();
412        let producer = self.producer.clone();
413        let result = tokio::task::spawn_blocking(move || {
414            producer
415                .client()
416                .fetch_metadata(None, Duration::from_secs(2))
417        })
418        .await;
419        let healthy = matches!(result, Ok(Ok(_)));
420        self.record("health", healthy, started_at).await;
421        if healthy {
422            nidus_http::health::HealthStatus::up()
423        } else {
424            nidus_http::health::HealthStatus::down("kafka metadata check failed")
425        }
426    }
427
428    /// Adds this provider as a readiness check on a health registry.
429    #[cfg(feature = "health")]
430    pub fn register_ready_check(
431        self: std::sync::Arc<Self>,
432        registry: nidus_http::health::HealthRegistry,
433        name: impl Into<String>,
434    ) -> nidus_http::health::HealthRegistry {
435        registry.ready_check(name, move || {
436            let provider = std::sync::Arc::clone(&self);
437            async move { provider.health_status().await }
438        })
439    }
440
441    async fn publish_with_headers(
442        &self,
443        topic: &str,
444        key: Option<&[u8]>,
445        payload: &[u8],
446        headers: Option<OwnedHeaders>,
447    ) -> Result<KafkaDelivery> {
448        if topic.trim().is_empty() {
449            return Err(KafkaError::Configuration {
450                message: "Kafka topic cannot be empty",
451            });
452        }
453        if self.shutting_down.load(Ordering::Acquire) {
454            return Err(KafkaError::ShuttingDown);
455        }
456        let started_at = Instant::now();
457        let mut record = FutureRecord::to(topic).payload(payload);
458        if let Some(key) = key {
459            record = record.key(key);
460        }
461        if let Some(headers) = headers {
462            record = record.headers(headers);
463        }
464        let result = self
465            .producer
466            .send(record, self.config.enqueue_timeout)
467            .await;
468        self.record("publish", result.is_ok(), started_at).await;
469        match result {
470            Ok(delivery) => Ok(KafkaDelivery {
471                partition: delivery.partition,
472                offset: delivery.offset,
473            }),
474            Err((source, message)) => {
475                let _ = message.payload();
476                Err(KafkaError::Delivery { source })
477            }
478        }
479    }
480
481    async fn record(&self, operation: &'static str, success: bool, started_at: Instant) {
482        self.telemetry
483            .record(&IntegrationEvent::new(
484                "nidus-kafka",
485                operation,
486                if success {
487                    IntegrationStatus::Success
488                } else {
489                    IntegrationStatus::Failure
490                },
491                started_at.elapsed(),
492            ))
493            .await;
494    }
495}
496
497impl fmt::Debug for KafkaProvider {
498    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
499        formatter
500            .debug_struct("KafkaProvider")
501            .field("config", &self.config)
502            .field("producer", &"FutureProducer")
503            .field("admin", &"AdminClient")
504            .field("shutting_down", &self.shutting_down.load(Ordering::Acquire))
505            .field("telemetry", &self.telemetry)
506            .finish()
507    }
508}
509
510#[async_trait]
511impl LifecycleHook for KafkaProvider {
512    async fn on_shutdown(&self) -> nidus_core::Result<()> {
513        self.shutdown()
514            .await
515            .map_err(|_| NidusError::ApplicationBuild {
516                message: "Kafka producer flush failed during shutdown".to_owned(),
517            })
518    }
519}