1#![forbid(unsafe_code)]
2#![deny(missing_docs)]
3
4use std::{
12 collections::BTreeMap,
13 fmt,
14 sync::{
15 Arc,
16 atomic::{AtomicBool, Ordering},
17 },
18 time::{Duration, Instant},
19};
20
21use async_trait::async_trait;
22use nidus_core::{Container, LifecycleHook, NidusError};
23use nidus_integrations::{
24 IntegrationEvent, IntegrationStatus, IntegrationTelemetry, MessageEnvelope,
25};
26use rdkafka::{
27 ClientConfig, Message,
28 admin::AdminClient,
29 client::DefaultClientContext,
30 consumer::StreamConsumer,
31 error::KafkaError as NativeKafkaError,
32 message::{Header, OwnedHeaders},
33 producer::{FutureProducer, FutureRecord, Producer},
34};
35use serde::Serialize;
36use thiserror::Error;
37
38pub type Result<T> = std::result::Result<T, KafkaError>;
40
41#[derive(Debug, Error)]
43pub enum KafkaError {
44 #[error(transparent)]
46 Kafka(#[from] NativeKafkaError),
47 #[error("Kafka delivery failed: {source}")]
49 Delivery {
50 #[source]
52 source: NativeKafkaError,
53 },
54 #[error("Kafka blocking task failed")]
56 TaskJoin,
57 #[error(transparent)]
59 Integration(#[from] nidus_integrations::IntegrationError),
60 #[error(transparent)]
62 Nidus(#[from] NidusError),
63 #[error("invalid Kafka configuration: {message}")]
65 Configuration {
66 message: &'static str,
68 },
69 #[error("Kafka provider is shutting down")]
71 ShuttingDown,
72}
73
74#[derive(Clone, Eq, PartialEq)]
76pub struct KafkaConfig {
77 bootstrap_servers: String,
78 client_id: String,
79 properties: BTreeMap<String, String>,
80 enqueue_timeout: Duration,
81 shutdown_timeout: Duration,
82 allow_plaintext_local: bool,
83}
84
85impl KafkaConfig {
86 pub fn new(bootstrap_servers: impl Into<String>, client_id: impl Into<String>) -> Self {
88 Self {
89 bootstrap_servers: bootstrap_servers.into(),
90 client_id: client_id.into(),
91 properties: BTreeMap::from([
92 ("security.protocol".to_owned(), "SSL".to_owned()),
93 ("enable.idempotence".to_owned(), "true".to_owned()),
94 ("acks".to_owned(), "all".to_owned()),
95 ("message.timeout.ms".to_owned(), "30000".to_owned()),
96 (
97 "queue.buffering.max.messages".to_owned(),
98 "100000".to_owned(),
99 ),
100 ]),
101 enqueue_timeout: Duration::from_secs(5),
102 shutdown_timeout: Duration::from_secs(10),
103 allow_plaintext_local: false,
104 }
105 }
106
107 pub fn property(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
109 self.properties.insert(key.into(), value.into());
110 self
111 }
112
113 pub fn allow_plaintext_for_local_development(mut self) -> Self {
115 self.properties
116 .insert("security.protocol".to_owned(), "PLAINTEXT".to_owned());
117 self.allow_plaintext_local = true;
118 self
119 }
120
121 pub fn with_enqueue_timeout(mut self, value: Duration) -> Self {
123 self.enqueue_timeout = value;
124 self
125 }
126
127 pub fn with_shutdown_timeout(mut self, value: Duration) -> Self {
129 self.shutdown_timeout = value;
130 self
131 }
132
133 pub fn bootstrap_servers(&self) -> &str {
135 &self.bootstrap_servers
136 }
137
138 pub fn client_id(&self) -> &str {
140 &self.client_id
141 }
142
143 pub fn properties(&self) -> &BTreeMap<String, String> {
145 &self.properties
146 }
147
148 pub fn validate(&self) -> Result<()> {
150 if self.bootstrap_servers.trim().is_empty() {
151 return Err(KafkaError::Configuration {
152 message: "bootstrap_servers cannot be empty",
153 });
154 }
155 if self.client_id.trim().is_empty() {
156 return Err(KafkaError::Configuration {
157 message: "client_id cannot be empty",
158 });
159 }
160 if self.enqueue_timeout.is_zero() || self.shutdown_timeout.is_zero() {
161 return Err(KafkaError::Configuration {
162 message: "Kafka timeouts must be greater than zero",
163 });
164 }
165 let protocol = self
166 .properties
167 .get("security.protocol")
168 .map(String::as_str)
169 .unwrap_or("PLAINTEXT");
170 let plaintext = protocol.eq_ignore_ascii_case("PLAINTEXT")
171 || protocol.eq_ignore_ascii_case("SASL_PLAINTEXT");
172 if plaintext && !self.allow_plaintext_local {
173 return Err(KafkaError::Configuration {
174 message: "plaintext Kafka requires an explicit local-development opt-in",
175 });
176 }
177 if plaintext && !kafka_bootstrap_is_loopback(&self.bootstrap_servers) {
178 return Err(KafkaError::Configuration {
179 message: "plaintext Kafka is restricted to loopback brokers",
180 });
181 }
182 Ok(())
183 }
184
185 fn native_config(&self) -> Result<ClientConfig> {
186 self.validate()?;
187 let mut config = ClientConfig::new();
188 config
189 .set("bootstrap.servers", &self.bootstrap_servers)
190 .set("client.id", &self.client_id);
191 for (key, value) in &self.properties {
192 config.set(key, value);
193 }
194 Ok(config)
195 }
196}
197
198fn kafka_bootstrap_is_loopback(bootstrap_servers: &str) -> bool {
199 bootstrap_servers
200 .split(',')
201 .all(|server| kafka_broker_is_loopback(server.trim()))
202}
203
204fn kafka_broker_is_loopback(server: &str) -> bool {
205 if server.is_empty() || server.contains('@') {
206 return false;
207 }
208 let Some((host, port)) = server.rsplit_once(':') else {
209 return false;
210 };
211 if port.parse::<u16>().is_err() {
212 return false;
213 }
214 let host = host
215 .strip_prefix('[')
216 .and_then(|host| host.strip_suffix(']'))
217 .unwrap_or(host);
218 host.eq_ignore_ascii_case("localhost")
219 || host
220 .parse::<std::net::IpAddr>()
221 .is_ok_and(|address| address.is_loopback())
222}
223
224impl fmt::Debug for KafkaConfig {
225 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
226 formatter
227 .debug_struct("KafkaConfig")
228 .field("bootstrap_servers", &"<redacted>")
229 .field("client_id", &self.client_id)
230 .field(
231 "property_names",
232 &self.properties.keys().collect::<Vec<_>>(),
233 )
234 .field("enqueue_timeout", &self.enqueue_timeout)
235 .field("shutdown_timeout", &self.shutdown_timeout)
236 .field("allow_plaintext_local", &self.allow_plaintext_local)
237 .finish()
238 }
239}
240
241#[derive(Clone, Copy, Debug, Eq, PartialEq)]
243pub struct KafkaDelivery {
244 partition: i32,
245 offset: i64,
246}
247
248impl KafkaDelivery {
249 pub const fn partition(&self) -> i32 {
251 self.partition
252 }
253
254 pub const fn offset(&self) -> i64 {
256 self.offset
257 }
258}
259
260#[derive(Clone, Debug)]
262pub struct KafkaProviderBuilder {
263 config: KafkaConfig,
264 telemetry: IntegrationTelemetry,
265}
266
267impl KafkaProviderBuilder {
268 pub fn new(config: KafkaConfig) -> Self {
270 Self {
271 config,
272 telemetry: IntegrationTelemetry::new(),
273 }
274 }
275
276 pub fn telemetry(mut self, telemetry: IntegrationTelemetry) -> Self {
278 self.telemetry = telemetry;
279 self
280 }
281
282 pub fn build(self) -> Result<KafkaProvider> {
284 let native = self.config.native_config()?;
285 let producer = native.create::<FutureProducer>()?;
286 let admin = native.create::<AdminClient<DefaultClientContext>>()?;
287 Ok(KafkaProvider {
288 config: self.config,
289 producer,
290 admin: Arc::new(admin),
291 shutting_down: Arc::new(AtomicBool::new(false)),
292 telemetry: self.telemetry,
293 })
294 }
295
296 pub fn register(self, container: &mut Container) -> Result<()> {
298 container.register_singleton(self.build()?)?;
299 Ok(())
300 }
301}
302
303#[derive(Clone)]
305pub struct KafkaProvider {
306 config: KafkaConfig,
307 producer: FutureProducer,
308 admin: Arc<AdminClient<DefaultClientContext>>,
309 shutting_down: Arc<AtomicBool>,
310 telemetry: IntegrationTelemetry,
311}
312
313impl KafkaProvider {
314 pub fn builder(config: KafkaConfig) -> KafkaProviderBuilder {
316 KafkaProviderBuilder::new(config)
317 }
318
319 pub fn producer(&self) -> &FutureProducer {
321 &self.producer
322 }
323
324 pub fn admin(&self) -> &AdminClient<DefaultClientContext> {
326 &self.admin
327 }
328
329 pub fn consumer(&self, group_id: &str) -> Result<StreamConsumer> {
334 if group_id.trim().is_empty() {
335 return Err(KafkaError::Configuration {
336 message: "Kafka consumer group_id cannot be empty",
337 });
338 }
339 let mut config = self.config.native_config()?;
340 config
341 .set("group.id", group_id)
342 .set("enable.auto.commit", "false")
343 .set("enable.auto.offset.store", "false")
344 .set("isolation.level", "read_committed");
345 Ok(config.create()?)
346 }
347
348 pub async fn publish(
350 &self,
351 topic: &str,
352 key: Option<&[u8]>,
353 payload: &[u8],
354 ) -> Result<KafkaDelivery> {
355 self.publish_with_headers(topic, key, payload, None).await
356 }
357
358 pub async fn publish_envelope<T>(
360 &self,
361 topic: &str,
362 key: Option<&[u8]>,
363 envelope: &MessageEnvelope<T>,
364 ) -> Result<KafkaDelivery>
365 where
366 T: Serialize,
367 {
368 let payload = envelope.to_json()?;
369 let mut headers = OwnedHeaders::new()
370 .insert(Header {
371 key: "nidus-message-id",
372 value: Some(envelope.id()),
373 })
374 .insert(Header {
375 key: "nidus-message-name",
376 value: Some(envelope.name()),
377 });
378 if let Some(value) = envelope.metadata().correlation_id_value() {
379 headers = headers.insert(Header {
380 key: "nidus-correlation-id",
381 value: Some(value),
382 });
383 }
384 if let Some(value) = envelope.metadata().traceparent_value() {
385 headers = headers.insert(Header {
386 key: "traceparent",
387 value: Some(value),
388 });
389 }
390 self.publish_with_headers(topic, key, &payload, Some(headers))
391 .await
392 }
393
394 pub async fn shutdown(&self) -> Result<()> {
396 self.shutting_down.store(true, Ordering::Release);
397 let producer = self.producer.clone();
398 let timeout = self.config.shutdown_timeout;
399 tokio::task::spawn_blocking(move || producer.flush(timeout))
400 .await
401 .map_err(|_| KafkaError::TaskJoin)??;
402 Ok(())
403 }
404
405 #[cfg(feature = "health")]
407 pub async fn health_status(&self) -> nidus_http::health::HealthStatus {
408 if self.shutting_down.load(Ordering::Acquire) {
409 return nidus_http::health::HealthStatus::down("kafka provider is shutting down");
410 }
411 let started_at = Instant::now();
412 let producer = self.producer.clone();
413 let result = tokio::task::spawn_blocking(move || {
414 producer
415 .client()
416 .fetch_metadata(None, Duration::from_secs(2))
417 })
418 .await;
419 let healthy = matches!(result, Ok(Ok(_)));
420 self.record("health", healthy, started_at).await;
421 if healthy {
422 nidus_http::health::HealthStatus::up()
423 } else {
424 nidus_http::health::HealthStatus::down("kafka metadata check failed")
425 }
426 }
427
428 #[cfg(feature = "health")]
430 pub fn register_ready_check(
431 self: std::sync::Arc<Self>,
432 registry: nidus_http::health::HealthRegistry,
433 name: impl Into<String>,
434 ) -> nidus_http::health::HealthRegistry {
435 registry.ready_check(name, move || {
436 let provider = std::sync::Arc::clone(&self);
437 async move { provider.health_status().await }
438 })
439 }
440
441 async fn publish_with_headers(
442 &self,
443 topic: &str,
444 key: Option<&[u8]>,
445 payload: &[u8],
446 headers: Option<OwnedHeaders>,
447 ) -> Result<KafkaDelivery> {
448 if topic.trim().is_empty() {
449 return Err(KafkaError::Configuration {
450 message: "Kafka topic cannot be empty",
451 });
452 }
453 if self.shutting_down.load(Ordering::Acquire) {
454 return Err(KafkaError::ShuttingDown);
455 }
456 let started_at = Instant::now();
457 let mut record = FutureRecord::to(topic).payload(payload);
458 if let Some(key) = key {
459 record = record.key(key);
460 }
461 if let Some(headers) = headers {
462 record = record.headers(headers);
463 }
464 let result = self
465 .producer
466 .send(record, self.config.enqueue_timeout)
467 .await;
468 self.record("publish", result.is_ok(), started_at).await;
469 match result {
470 Ok(delivery) => Ok(KafkaDelivery {
471 partition: delivery.partition,
472 offset: delivery.offset,
473 }),
474 Err((source, message)) => {
475 let _ = message.payload();
476 Err(KafkaError::Delivery { source })
477 }
478 }
479 }
480
481 async fn record(&self, operation: &'static str, success: bool, started_at: Instant) {
482 self.telemetry
483 .record(&IntegrationEvent::new(
484 "nidus-kafka",
485 operation,
486 if success {
487 IntegrationStatus::Success
488 } else {
489 IntegrationStatus::Failure
490 },
491 started_at.elapsed(),
492 ))
493 .await;
494 }
495}
496
497impl fmt::Debug for KafkaProvider {
498 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
499 formatter
500 .debug_struct("KafkaProvider")
501 .field("config", &self.config)
502 .field("producer", &"FutureProducer")
503 .field("admin", &"AdminClient")
504 .field("shutting_down", &self.shutting_down.load(Ordering::Acquire))
505 .field("telemetry", &self.telemetry)
506 .finish()
507 }
508}
509
510#[async_trait]
511impl LifecycleHook for KafkaProvider {
512 async fn on_shutdown(&self) -> nidus_core::Result<()> {
513 self.shutdown()
514 .await
515 .map_err(|_| NidusError::ApplicationBuild {
516 message: "Kafka producer flush failed during shutdown".to_owned(),
517 })
518 }
519}