use faucet_common_rabbitmq::{RabbitMqConnectionConfig, RabbitMqExchangeKind, RabbitMqValueFormat};
use faucet_core::{DEFAULT_BATCH_SIZE, FaucetError};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
fn default_batch_size() -> usize {
DEFAULT_BATCH_SIZE
}
fn default_true() -> bool {
true
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum AckMode {
#[default]
OnSinkConfirm,
Auto,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum OnDecodeError {
#[default]
Fail,
Skip,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct RabbitMqBinding {
pub exchange: String,
#[serde(default)]
pub routing_key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub exchange_kind: Option<RabbitMqExchangeKind>,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct RabbitMqSourceConfig {
#[serde(flatten)]
pub connection: RabbitMqConnectionConfig,
pub queue: String,
#[serde(default = "default_true")]
pub declare_queue: bool,
#[serde(default = "default_true")]
pub queue_durable: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub bindings: Vec<RabbitMqBinding>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prefetch: Option<u16>,
#[serde(default)]
pub ack_mode: AckMode,
#[serde(default)]
pub value_format: RabbitMqValueFormat,
#[serde(default)]
pub on_decode_error: OnDecodeError,
#[serde(default)]
pub include_metadata: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub consumer_tag: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_messages: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub idle_timeout_secs: Option<u64>,
#[serde(default = "default_batch_size")]
pub batch_size: usize,
}
impl RabbitMqSourceConfig {
pub fn new(queue: impl Into<String>) -> Self {
Self {
connection: RabbitMqConnectionConfig::default(),
queue: queue.into(),
declare_queue: true,
queue_durable: true,
bindings: Vec::new(),
prefetch: None,
ack_mode: AckMode::default(),
value_format: RabbitMqValueFormat::default(),
on_decode_error: OnDecodeError::default(),
include_metadata: false,
consumer_tag: None,
max_messages: None,
idle_timeout_secs: Some(5),
batch_size: DEFAULT_BATCH_SIZE,
}
}
pub fn effective_prefetch(&self) -> u16 {
match self.prefetch {
Some(p) => p,
None if self.batch_size == 0 => 0,
None => u16::try_from(self.batch_size).unwrap_or(0),
}
}
pub fn validate(&self) -> Result<(), FaucetError> {
self.connection.validate()?;
if self.queue.trim().is_empty() {
return Err(FaucetError::Config(
"rabbitmq source: `queue` must not be empty".into(),
));
}
for b in &self.bindings {
if b.exchange.trim().is_empty() {
return Err(FaucetError::Config(
"rabbitmq source: a binding's `exchange` must not be empty (the default \
exchange cannot be bound)"
.into(),
));
}
}
if self.max_messages.is_none() && self.idle_timeout_secs.is_none() {
return Err(FaucetError::Config(
"rabbitmq source: at least one of `max_messages` or `idle_timeout_secs` must be \
set so the run terminates"
.into(),
));
}
if self.max_messages == Some(0) {
return Err(FaucetError::Config(
"rabbitmq source: `max_messages` must be greater than 0".into(),
));
}
if self.idle_timeout_secs == Some(0) {
return Err(FaucetError::Config(
"rabbitmq source: `idle_timeout_secs` must be greater than 0".into(),
));
}
faucet_core::validate_batch_size(self.batch_size)?;
if self.ack_mode == AckMode::Auto && self.max_messages.is_some() {
return Err(FaucetError::Config(
"rabbitmq source: `ack_mode: auto` cannot be combined with `max_messages` — under \
`auto` the broker settles every message it pushes, and it pushes the backlog \
into the client buffer ahead of consumption, so everything buffered past \
`max_messages` would be lost when the run stops; use `ack_mode: \
on_sink_confirm`, or terminate on `idle_timeout_secs` alone"
.into(),
));
}
if self.ack_mode == AckMode::OnSinkConfirm {
let prefetch = self.effective_prefetch();
let too_small =
prefetch != 0 && (self.batch_size == 0 || usize::from(prefetch) < self.batch_size);
if too_small {
return Err(FaucetError::Config(format!(
"rabbitmq source: `prefetch` ({prefetch}) is below `batch_size` ({}) under \
`ack_mode: on_sink_confirm` — the broker would stop delivering before a \
page fills; raise `prefetch`, set it to 0 (unlimited), or lower `batch_size`",
if self.batch_size == 0 {
"0 = whole run".to_string()
} else {
self.batch_size.to_string()
}
)));
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn auto_ack_with_max_messages_is_refused() {
let mut c = RabbitMqSourceConfig::new("orders");
c.ack_mode = AckMode::Auto;
c.idle_timeout_secs = Some(5);
c.validate().unwrap();
c.max_messages = Some(10);
let err = c.validate().unwrap_err().to_string();
assert!(err.contains("ack_mode: auto"), "{err}");
c.ack_mode = AckMode::OnSinkConfirm;
c.validate().unwrap();
}
#[test]
fn minimal_is_valid() {
let c = RabbitMqSourceConfig::new("orders");
assert!(c.validate().is_ok());
assert_eq!(c.effective_prefetch(), 1000);
}
#[test]
fn deserializes_with_defaults() {
let c: RabbitMqSourceConfig = serde_json::from_value(json!({
"url": "amqp://broker",
"queue": "q",
"max_messages": 10
}))
.unwrap();
assert!(c.declare_queue && c.queue_durable);
assert_eq!(c.ack_mode, AckMode::OnSinkConfirm);
assert_eq!(c.value_format, RabbitMqValueFormat::Json);
assert_eq!(c.on_decode_error, OnDecodeError::Fail);
assert_eq!(c.batch_size, DEFAULT_BATCH_SIZE);
assert!(c.validate().is_ok());
}
#[test]
fn deserializes_bindings_and_modes() {
let c: RabbitMqSourceConfig = serde_json::from_value(json!({
"queue": "q",
"idle_timeout_secs": 2,
"ack_mode": "auto",
"on_decode_error": "skip",
"value_format": "bytes",
"bindings": [{"exchange": "events", "routing_key": "a.*", "exchange_kind": "topic"}]
}))
.unwrap();
assert_eq!(c.ack_mode, AckMode::Auto);
assert_eq!(c.on_decode_error, OnDecodeError::Skip);
assert_eq!(
c.bindings[0].exchange_kind,
Some(RabbitMqExchangeKind::Topic)
);
assert!(c.validate().is_ok());
}
#[test]
fn rejects_empty_queue_and_binding_exchange() {
let mut c = RabbitMqSourceConfig::new(" ");
assert!(c.validate().unwrap_err().to_string().contains("queue"));
c.queue = "q".into();
c.bindings.push(RabbitMqBinding {
exchange: "".into(),
routing_key: "".into(),
exchange_kind: None,
});
assert!(c.validate().unwrap_err().to_string().contains("binding"));
}
#[test]
fn requires_a_terminator() {
let mut c = RabbitMqSourceConfig::new("q");
c.idle_timeout_secs = None;
assert!(
c.validate()
.unwrap_err()
.to_string()
.contains("max_messages")
);
c.max_messages = Some(0);
assert!(
c.validate()
.unwrap_err()
.to_string()
.contains("greater than 0")
);
c.max_messages = Some(5);
assert!(c.validate().is_ok());
c.idle_timeout_secs = Some(0);
assert!(
c.validate()
.unwrap_err()
.to_string()
.contains("idle_timeout")
);
}
#[test]
fn rejects_batch_size_over_max() {
let mut c = RabbitMqSourceConfig::new("q");
c.batch_size = faucet_core::MAX_BATCH_SIZE + 1;
assert!(matches!(c.validate(), Err(FaucetError::Config(_))));
}
#[test]
fn prefetch_rules_under_on_sink_confirm() {
let mut c = RabbitMqSourceConfig::new("q");
c.prefetch = Some(10);
assert!(c.validate().unwrap_err().to_string().contains("prefetch"));
c.prefetch = Some(0);
assert!(c.validate().is_ok());
c.prefetch = Some(5000);
assert!(c.validate().is_ok());
c.prefetch = None;
c.batch_size = 0;
assert_eq!(c.effective_prefetch(), 0);
assert!(c.validate().is_ok());
c.prefetch = Some(100);
assert!(c.validate().unwrap_err().to_string().contains("whole run"));
c.prefetch = None;
c.batch_size = 100_000;
assert_eq!(c.effective_prefetch(), 0);
assert!(c.validate().is_ok());
c.ack_mode = AckMode::Auto;
c.prefetch = Some(1);
assert!(c.validate().is_ok());
}
#[test]
fn connection_errors_propagate() {
let mut c = RabbitMqSourceConfig::new("q");
c.connection.connect_timeout_secs = 0;
assert!(c.validate().is_err());
}
#[test]
fn schema_compiles() {
let _ = schemars::schema_for!(RabbitMqSourceConfig);
}
}