1use std::sync::Arc;
2use std::time::Duration;
3
4use async_trait::async_trait;
5use bytes::Bytes;
6use camel_api::datasource::DatasourceCatalog;
7use futures::TryStreamExt;
8use serde_json::Value as JsonValue;
9use sqlx::AnyPool;
10use sqlx::any::AnyPoolOptions;
11use sqlx::any::AnyRow;
12use tokio::sync::OnceCell;
13use tracing::{debug, error, info, warn};
14
15use camel_component_api::retry_async;
16use camel_component_api::{
17 Body, CamelError, Exchange, Message, RuntimeObservability, StreamBody, StreamMetadata,
18};
19use camel_component_api::{ConcurrencyModel, Consumer, ConsumerContext};
20
21use crate::config::{
22 PollStrategy, ProcessingStrategy, SqlEndpointConfig, SqlOutputType, TransactionMode,
23 enrich_db_url_with_ssl, redact_db_url,
24};
25use crate::headers;
26use crate::query::{QueryTemplate, parse_query_template, resolve_params};
27use crate::utils::{bind_json_values, is_retryable_sqlx_error, row_to_json};
28
29fn record_post_process_failure(
43 runtime: &dyn RuntimeObservability,
44 route_id: &str,
45 label: &str,
46 error: &CamelError,
47 message: &str,
48) {
49 runtime.metrics().increment_errors(route_id, label);
51 error!(error = %error, "{message}");
53}
54
55#[derive(Debug, Clone, Copy, Default)]
59struct PollOutcome {
60 was_empty: bool,
61}
62
63pub struct SqlConsumer {
64 pub(crate) config: SqlEndpointConfig,
65 pub(crate) pool: Arc<OnceCell<Arc<AnyPool>>>,
66 pub(crate) catalog: Option<Arc<dyn DatasourceCatalog>>,
67 stopped: bool,
68 runtime: Arc<dyn RuntimeObservability>,
71}
72
73impl SqlConsumer {
74 pub fn new(
75 config: SqlEndpointConfig,
76 pool: Arc<OnceCell<Arc<AnyPool>>>,
77 catalog: Option<Arc<dyn DatasourceCatalog>>,
78 runtime: Arc<dyn RuntimeObservability>,
79 ) -> Self {
80 Self {
81 config,
82 pool,
83 catalog,
84 stopped: false,
85 runtime,
86 }
87 }
88
89 async fn poll_database(
91 &self,
92 pool: &AnyPool,
93 context: &ConsumerContext,
94 template: &QueryTemplate,
95 ) -> Result<PollOutcome, CamelError> {
96 let route_id = context.route_id();
98
99 let empty_exchange = Exchange::new(Message::default());
101
102 let prepared = resolve_params(template, &empty_exchange, &self.config.in_separator)?;
104
105 debug!(query = %prepared.sql, "executing SQL consumer poll");
106
107 if self.config.output_type == SqlOutputType::StreamList {
108 return self.poll_database_stream(pool, context, &prepared).await;
109 }
110
111 let query = bind_json_values(sqlx::query(&prepared.sql), &prepared.bindings);
112 let rows: Vec<AnyRow> = query.fetch_all(pool).await.map_err(|e| {
113 warn!(error = %e, "SQL consumer poll query failed");
114 CamelError::ProcessorError(format!("Query execution failed: {}", e))
115 })?;
116
117 debug!(rows = rows.len(), "SQL consumer poll completed");
118
119 let was_empty = rows.is_empty();
120 if was_empty && !self.config.route_empty_result_set {
121 return Ok(PollOutcome { was_empty });
122 }
123
124 let rows_to_process: Vec<AnyRow> = if let Some(max) = self.config.max_messages_per_poll {
125 if max > 0 {
126 rows.into_iter().take(max as usize).collect()
127 } else {
128 rows
129 }
130 } else {
131 rows
132 };
133
134 if self.config.use_iterator {
135 for row in rows_to_process {
137 let row_json = row_to_json(&row)?;
138
139 let mut msg = Message::new(Body::Json(row_json.clone()));
141
142 if let Some(obj) = row_json.as_object() {
144 for (key, value) in obj {
145 msg.set_header(format!("CamelSql.{}", key), value.clone());
146 }
147 }
148
149 let exchange = Exchange::new(msg);
150
151 let result = context.send_and_wait(exchange).await;
153
154 if let Err(e) = self.handle_post_processing(pool, &result, &row_json).await {
156 record_post_process_failure(
157 self.runtime.as_ref(),
158 route_id,
159 "b-prime:sql:on-consume",
160 &e,
161 "Post-processing failed",
162 );
163 if self.config.break_batch_on_consume_fail {
164 return Err(e);
165 }
166 }
167
168 if let Err(ref consume_err) = result
170 && self.config.break_batch_on_consume_fail
171 {
172 return Err(consume_err.clone());
173 }
174 }
175 } else {
176 let rows_json: Vec<JsonValue> = rows_to_process
178 .iter()
179 .map(row_to_json)
180 .collect::<Result<Vec<_>, CamelError>>()?;
181
182 let row_count = rows_json.len();
183
184 let mut msg = Message::new(Body::Json(JsonValue::Array(rows_json.clone())));
186 msg.set_header(headers::ROW_COUNT, JsonValue::Number(row_count.into()));
187
188 let exchange = Exchange::new(msg);
189
190 let result = context.send_and_wait(exchange).await;
192
193 for row_json in rows_json.iter() {
197 if let Err(e) = self.handle_post_processing(pool, &result, row_json).await {
198 record_post_process_failure(
199 self.runtime.as_ref(),
200 route_id,
201 "b-prime:sql:on-consume-batch",
202 &e,
203 "Post-processing failed for batch row",
204 );
205 if self.config.break_batch_on_consume_fail {
206 return Err(e);
207 }
208 }
209 }
210
211 if let Err(ref consume_err) = result
213 && self.config.break_batch_on_consume_fail
214 {
215 return Err(consume_err.clone());
216 }
217 }
218
219 if let Some(ref batch_query) = self.config.on_consume_batch_complete {
221 let _ = self
222 .execute_post_query(pool, batch_query, &JsonValue::Null)
223 .await;
224 }
225
226 Ok(PollOutcome { was_empty })
227 }
228
229 async fn poll_database_stream(
230 &self,
231 pool: &AnyPool,
232 context: &ConsumerContext,
233 prepared: &crate::query::PreparedQuery,
234 ) -> Result<PollOutcome, CamelError> {
235 let pool_clone = pool.clone();
236 let sql_str = prepared.sql.clone();
237 let bindings = prepared.bindings.clone();
238
239 let byte_stream = async_stream::try_stream! {
240 let mut q = sqlx::query(&sql_str);
241 q = bind_json_values(q, &bindings);
242 let mut rows = q.fetch(&pool_clone);
243 while let Some(row) = rows.try_next().await.map_err(|e| {
244 CamelError::ProcessorError(format!("Query execution failed: {}", e))
245 })? {
246 let json_val = row_to_json(&row).map_err(|e| {
247 CamelError::ProcessorError(format!("JSON serialization failed: {}", e))
248 })?;
249 let mut bytes = serde_json::to_vec(&json_val)
250 .map_err(|e| CamelError::ProcessorError(format!("JSON serialization failed: {}", e)))?;
251 bytes.push(b'\n');
252 yield Bytes::from(bytes);
253 }
254 };
255
256 let msg = Message::new(Body::Stream(StreamBody {
257 stream: Arc::new(tokio::sync::Mutex::new(Some(Box::pin(byte_stream)))),
258 metadata: StreamMetadata {
259 content_type: Some("application/x-ndjson".to_string()),
260 size_hint: None,
261 origin: None,
262 },
263 }));
264
265 let exchange = Exchange::new(msg);
266 let result = context.send_and_wait(exchange).await;
267 if let Err(e) = result {
268 record_post_process_failure(
269 self.runtime.as_ref(),
270 context.route_id(),
271 "b-prime:sql:stream-list",
272 &e,
273 "StreamList consumer downstream processing failed",
274 );
275 return Err(e);
276 }
277
278 debug!("StreamList: consumer poll completed (lazy stream emitted)");
279 Ok(PollOutcome::default())
281 }
282
283 async fn handle_post_processing(
285 &self,
286 pool: &AnyPool,
287 result: &Result<Exchange, CamelError>,
288 row_json: &JsonValue,
289 ) -> Result<(), CamelError> {
290 match result {
291 Ok(_) => {
292 if let Some(ref on_consume) = self.config.on_consume {
294 self.execute_post_query(pool, on_consume, row_json).await?;
295 }
296 }
297 Err(_) => {
298 if let Some(ref on_consume_failed) = self.config.on_consume_failed {
300 self.execute_post_query(pool, on_consume_failed, row_json)
301 .await?;
302 }
303 }
304 }
305 Ok(())
306 }
307
308 async fn execute_post_query(
310 &self,
311 pool: &AnyPool,
312 query_str: &str,
313 row_json: &JsonValue,
314 ) -> Result<(), CamelError> {
315 let template = parse_query_template(query_str, self.config.placeholder)?;
317
318 let mut temp_msg = Message::new(Body::Json(row_json.clone()));
321 if let Some(obj) = row_json.as_object() {
322 for (key, value) in obj {
323 temp_msg.set_header(format!("CamelSql.{}", key), value.clone());
324 }
325 }
326 let temp_exchange = Exchange::new(temp_msg);
327
328 let prepared = resolve_params(&template, &temp_exchange, &self.config.in_separator)?;
330
331 let query = bind_json_values(sqlx::query(&prepared.sql), &prepared.bindings);
333 let result = query.execute(pool).await.map_err(|e| {
334 CamelError::ProcessorError(format!("Post-query execution failed: {}", e))
335 })?;
336
337 if result.rows_affected() == 0 {
339 warn!(
340 query = query_str,
341 "Post-processing query affected 0 rows — the row may not have been marked correctly"
342 );
343 }
344
345 Ok(())
346 }
347
348 async fn handle_poll_result(
352 &self,
353 pool: &AnyPool,
354 context: &ConsumerContext,
355 template: &QueryTemplate,
356 ) -> PollOutcome {
357 match self.poll_database(pool, context, template).await {
358 Ok(outcome) => outcome,
359 Err(e) => {
360 if self.config.bridge_error_handler {
362 warn!(error = %e, "SQL consumer poll failed (bridged)");
366 if let Err(route_err) = self.bridge_poll_error(context, e).await {
367 error!(error = %route_err, "Failed to bridge SQL consumer error to route");
370 }
371 } else {
372 record_post_process_failure(
373 self.runtime.as_ref(),
374 context.route_id(),
375 "b-prime:sql:poll-failed",
376 &e,
377 "SQL consumer poll failed",
378 );
379 }
380 PollOutcome::default()
382 }
383 }
384 }
385
386 async fn bridge_poll_error(
387 &self,
388 context: &ConsumerContext,
389 error: CamelError,
390 ) -> Result<(), CamelError> {
391 if !self.config.bridge_error_handler {
392 return Ok(());
393 }
394 let mut exchange = Exchange::new(Message::default());
395 exchange.set_error(error);
396 context.send_and_wait(exchange).await.map(|_| ())
397 }
398}
399
400#[async_trait]
401impl Consumer for SqlConsumer {
402 async fn start(&mut self, context: ConsumerContext) -> Result<(), CamelError> {
403 if self.stopped {
405 return Err(CamelError::Config(
406 "SQL consumer cannot be restarted after stop".into(),
407 ));
408 }
409
410 let route_id = context.route_id().to_string();
412 let catalog = self.catalog.clone();
413 let ds_name = self.config.datasource_name.clone();
414
415 self.config.resolve_defaults();
417 self.config.resolve_file_query().await?;
418
419 let pool = self
420 .pool
421 .get_or_try_init(|| async {
422 if let (Some(ref cat), Some(ref name)) = (catalog, ds_name) {
424 let handle = cat.get_pool(name).await?;
425 return handle.downcast::<AnyPool>();
426 }
427
428 sqlx::any::install_default_drivers();
431 let db_url = enrich_db_url_with_ssl(&self.config.db_url, &self.config)?;
432
433 let max_conn = self.config.max_connections.ok_or_else(|| {
434 CamelError::Config("max_connections not resolved for SQL consumer pool".into())
435 })?;
436 let min_conn = self.config.min_connections.ok_or_else(|| {
437 CamelError::Config("min_connections not resolved for SQL consumer pool".into())
438 })?;
439 let idle_timeout = self.config.idle_timeout_secs.ok_or_else(|| {
440 CamelError::Config(
441 "idle_timeout_secs not resolved for SQL consumer pool".into(),
442 )
443 })?;
444 let max_lifetime = self.config.max_lifetime_secs.ok_or_else(|| {
445 CamelError::Config(
446 "max_lifetime_secs not resolved for SQL consumer pool".into(),
447 )
448 })?;
449
450 info!(
451 db_url = %redact_db_url(&self.config.db_url),
452 "SQL consumer pool initializing"
453 );
454 let retry_policy = &self.config.retry;
455 let pool = retry_async::<_, _, _, _, sqlx::Error>(
456 retry_policy,
457 "sql",
458 "consumer-pool-init",
459 || {
460 async {
461 AnyPoolOptions::new()
462 .max_connections(max_conn)
463 .min_connections(min_conn)
464 .idle_timeout(Duration::from_secs(idle_timeout))
465 .max_lifetime(Duration::from_secs(max_lifetime))
466 .connect(&db_url)
467 .await
468 }
469 },
470 is_retryable_sqlx_error,
471 Some(self.runtime.metrics().as_ref()),
472 )
473 .await
474 .map_err(|e| {
475 self.runtime.health().force_unhealthy_for_route(
476 &route_id,
477 "g:sql:consumer-pool-init",
478 &e.to_string(),
479 );
480 error!(error = %e, db_url = %redact_db_url(&self.config.db_url), "SQL connect failed, giving up");
482 CamelError::EndpointCreationFailed(format!(
483 "Failed to connect to database: {}",
484 e
485 ))
486 })?;
487 Ok(Arc::new(pool))
488 })
489 .await?;
490
491 if self.config.transaction_mode == TransactionMode::Managed {
493 warn!("transactionManager not yet implemented; using Auto mode");
494 }
495
496 if self.config.processing_strategy == ProcessingStrategy::Scheduled {
498 debug!(
499 "Processing strategy: Scheduled (rows dispatched individually via send_and_wait)"
500 );
501 }
502 if self.config.poll_strategy == PollStrategy::Burst {
503 debug!("Poll strategy: Burst (rapid successive polls)");
504 }
505
506 if self.config.output_type == SqlOutputType::StreamList
507 && (self.config.on_consume.is_some()
508 || self.config.on_consume_failed.is_some()
509 || self.config.on_consume_batch_complete.is_some()
510 || self.config.break_on_empty)
511 {
512 warn!(
513 "onConsume/onConsumeFailed/onConsumeBatchComplete/breakOnEmpty are not executed in \
514 StreamList mode (rows are consumed lazily downstream)"
515 );
516 }
517
518 if self.config.on_consume.is_none() {
520 warn!(
521 "SQL consumer started without onConsume configured — consumed rows will not be marked/deleted"
522 );
523 }
524
525 info!(
526 db_url = %redact_db_url(&self.config.db_url),
527 query_len = self.config.query.len(),
528 "SQL consumer started"
529 );
530
531 let template = parse_query_template(&self.config.query, self.config.placeholder)
533 .map_err(|e| CamelError::Config(format!("Invalid query template: {}", e)))?;
534
535 if self.config.initial_delay_ms > 0 {
537 tokio::select! {
538 _ = context.cancelled() => {
539 info!("SQL consumer stopped during initial delay");
540 return Ok(());
541 }
542 _ = tokio::time::sleep(Duration::from_millis(self.config.initial_delay_ms)) => {}
543 }
544 }
545
546 let mut poll_count: u32 = 0;
562 loop {
563 if let Some(max_repeats) = self.config.repeat_count
565 && poll_count >= max_repeats
566 {
567 info!(
568 repeat_count = max_repeats,
569 "SQL consumer reached repeat_count limit, stopping"
570 );
571 break;
572 }
573
574 tokio::select! {
575 _ = context.cancelled() => {
576 info!("SQL consumer stopped");
577 break;
578 }
579 _ = tokio::time::sleep(Duration::from_millis(self.config.delay_ms)) => {
580 poll_count += 1;
581 let outcome = self.handle_poll_result(pool.as_ref(), &context, &template).await;
582 if self.config.break_on_empty && outcome.was_empty {
583 info!("SQL consumer stopping: break_on_empty triggered (poll returned 0 rows)");
584 break;
585 }
586 }
587 }
588 }
589
590 Ok(())
591 }
592
593 async fn stop(&mut self) -> Result<(), CamelError> {
594 if self.stopped {
596 debug!("SQL consumer stop called on already-stopped consumer");
597 return Ok(());
598 }
599
600 if let Some(pool) = self.pool.get() {
602 debug!("SQL consumer closing connection pool");
603 pool.close().await;
604 debug!("SQL consumer pool closed");
605 }
606
607 self.stopped = true;
608 info!("SQL consumer stopped");
609 Ok(())
610 }
611
612 fn concurrency_model(&self) -> ConcurrencyModel {
613 ConcurrencyModel::Sequential
617 }
618}
619
620#[cfg(test)]
621mod tests {
622 use super::*;
623 use camel_api::MetricsCollector;
624 use camel_component_api::HealthCheckRegistry;
625 use camel_component_api::test_support::PanicRuntimeObservability;
626 fn test_rt() -> std::sync::Arc<dyn camel_component_api::RuntimeObservability> {
627 std::sync::Arc::new(PanicRuntimeObservability)
628 }
629 use crate::config::SqlEndpointConfig;
630 use camel_component_api::ExchangeEnvelope;
631 use camel_component_api::UriConfig;
632 use sqlx::any::AnyPoolOptions;
633 use std::sync::Arc;
634 use std::sync::Mutex;
635 use std::time::Duration;
636 use tokio::sync::mpsc;
637 use tokio::time::timeout;
638 use tokio_util::sync::CancellationToken;
639
640 struct RecordingMetrics {
645 errors: Arc<Mutex<Vec<(String, String)>>>,
646 }
647
648 impl MetricsCollector for RecordingMetrics {
649 fn record_exchange_duration(&self, _: &str, _: Duration) {}
650 fn increment_errors(&self, route_id: &str, error_type: &str) {
651 self.errors
652 .lock()
653 .unwrap()
654 .push((route_id.to_string(), error_type.to_string()));
655 }
656 fn increment_exchanges(&self, _: &str) {}
657 fn set_queue_depth(&self, _: &str, _: usize) {}
658 fn record_circuit_breaker_change(&self, _: &str, _: &str, _: &str) {}
659 }
660
661 struct RecordingRuntime {
662 metrics_collector: Arc<RecordingMetrics>,
663 }
664
665 impl RecordingRuntime {
666 fn new(errors: Arc<Mutex<Vec<(String, String)>>>) -> Self {
667 Self {
668 metrics_collector: Arc::new(RecordingMetrics { errors }),
669 }
670 }
671 }
672
673 impl RuntimeObservability for RecordingRuntime {
674 fn metrics(&self) -> Arc<dyn MetricsCollector> {
675 self.metrics_collector.clone() as Arc<dyn MetricsCollector>
676 }
677 fn health(&self) -> Arc<dyn HealthCheckRegistry> {
678 panic!("RecordingRuntime::health not used in this test")
679 }
680 }
681
682 #[tracing_test::traced_test]
686 #[test]
687 fn record_post_process_failure_increments_errors_and_emits_error_log() {
688 let errors: Arc<Mutex<Vec<(String, String)>>> = Arc::new(Mutex::new(Vec::new()));
689 let runtime = Arc::new(RecordingRuntime::new(Arc::clone(&errors)));
690 let error = CamelError::ProcessorError("test failure".to_string());
691
692 record_post_process_failure(
694 runtime.as_ref(),
695 "test-route",
696 "b-prime:sql:on-consume",
697 &error,
698 "Post-processing failed",
699 );
700
701 let recorded = errors.lock().unwrap();
703 assert_eq!(recorded.len(), 1, "expected 1 increment_errors call");
704 assert_eq!(recorded[0].0, "test-route");
705 assert_eq!(recorded[0].1, "b-prime:sql:on-consume");
706 drop(recorded);
707
708 assert!(logs_contain("ERROR"), "helper must emit error! log");
710 assert!(
711 logs_contain("Post-processing failed"),
712 "helper must include the message in the log"
713 );
714 }
715
716 async fn sqlite_pool() -> AnyPool {
717 sqlx::any::install_default_drivers();
718 AnyPoolOptions::new()
719 .max_connections(1)
720 .connect("sqlite::memory:")
721 .await
722 .expect("sqlite pool")
723 }
724
725 async fn seed_consumer_table(pool: &AnyPool) {
726 sqlx::query("CREATE TABLE jobs (id INTEGER PRIMARY KEY, processed INTEGER DEFAULT 0, failed INTEGER DEFAULT 0)")
727 .execute(pool)
728 .await
729 .expect("create table");
730 sqlx::query("INSERT INTO jobs (id, processed, failed) VALUES (1, 0, 0), (2, 0, 0)")
731 .execute(pool)
732 .await
733 .expect("seed rows");
734 }
735
736 fn config() -> SqlEndpointConfig {
737 let mut c =
738 SqlEndpointConfig::from_uri("sql:select * from t?db_url=postgres://localhost/test")
739 .unwrap();
740 c.resolve_defaults();
741 c
742 }
743
744 #[test]
745 fn consumer_concurrency_model() {
746 let c = SqlConsumer::new(config(), Arc::new(OnceCell::new()), None, test_rt());
747 assert_eq!(c.concurrency_model(), ConcurrencyModel::Sequential);
748 }
749
750 #[test]
751 fn consumer_stores_config() {
752 let mut config = SqlEndpointConfig::from_uri(
753 "sql:select * from t?db_url=postgres://localhost/test&delay=2000&onConsume=update t set done=true"
754 ).unwrap();
755 config.resolve_defaults();
756 let c = SqlConsumer::new(config.clone(), Arc::new(OnceCell::new()), None, test_rt());
757 assert_eq!(c.config.delay_ms, 2000);
758 assert!(c.config.on_consume.is_some());
759 }
760
761 #[tokio::test]
762 async fn poll_database_runs_on_consume_for_successful_rows() {
763 let pool = sqlite_pool().await;
764 seed_consumer_table(&pool).await;
765
766 let mut config = SqlEndpointConfig::from_uri(
767 "sql:select id, processed, failed from jobs where processed = 0 order by id?db_url=sqlite::memory:&onConsume=update jobs set processed=1 where id=:#id&initialDelay=0&delay=1",
768 )
769 .unwrap();
770 config.resolve_defaults();
771
772 let consumer = SqlConsumer::new(config.clone(), Arc::new(OnceCell::new()), None, test_rt());
773 let template = parse_query_template(&config.query, config.placeholder).unwrap();
774
775 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
776 tokio::spawn(async move {
777 loop {
778 match timeout(Duration::from_secs(2), rx.recv()).await {
779 Ok(Some(env)) => {
780 if let Some(reply_tx) = env.reply_tx {
781 let _ = reply_tx.send(Ok(env.exchange));
782 }
783 }
784 Ok(None) => break, Err(_) => break, }
787 }
788 });
789 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
790
791 consumer
792 .poll_database(&pool, &ctx, &template)
793 .await
794 .expect("poll must succeed");
795
796 let row = sqlx::query("select processed from jobs where id = 1")
797 .fetch_one(&pool)
798 .await
799 .expect("row 1");
800 let processed_1: i64 = sqlx::Row::try_get(&row, 0).expect("processed");
801
802 let row = sqlx::query("select processed from jobs where id = 2")
803 .fetch_one(&pool)
804 .await
805 .expect("row 2");
806 let processed_2: i64 = sqlx::Row::try_get(&row, 0).expect("processed");
807
808 assert_eq!(processed_1, 1);
809 assert_eq!(processed_2, 1);
810 }
811
812 #[tokio::test]
813 async fn poll_database_runs_on_consume_failed_when_downstream_fails() {
814 let pool = sqlite_pool().await;
815 seed_consumer_table(&pool).await;
816
817 let mut config = SqlEndpointConfig::from_uri(
818 "sql:select id, processed, failed from jobs where processed = 0 order by id?db_url=sqlite::memory:&onConsumeFailed=update jobs set failed=1 where id=:#id&initialDelay=0&delay=1",
819 )
820 .unwrap();
821 config.resolve_defaults();
822
823 let consumer = SqlConsumer::new(config.clone(), Arc::new(OnceCell::new()), None, test_rt());
824 let template = parse_query_template(&config.query, config.placeholder).unwrap();
825
826 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
827 tokio::spawn(async move {
828 loop {
829 match timeout(Duration::from_secs(2), rx.recv()).await {
830 Ok(Some(env)) => {
831 if let Some(reply_tx) = env.reply_tx {
832 let _ = reply_tx
833 .send(Err(CamelError::ProcessorError("downstream boom".into())));
834 }
835 }
836 Ok(None) => break, Err(_) => break, }
839 }
840 });
841 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
842
843 consumer
844 .poll_database(&pool, &ctx, &template)
845 .await
846 .expect("consumer should swallow downstream errors when breakBatchOnConsumeFail=false");
847
848 let row = sqlx::query("select failed from jobs where id = 1")
849 .fetch_one(&pool)
850 .await
851 .expect("row 1");
852 let failed_1: i64 = sqlx::Row::try_get(&row, 0).expect("failed");
853
854 let row = sqlx::query("select failed from jobs where id = 2")
855 .fetch_one(&pool)
856 .await
857 .expect("row 2");
858 let failed_2: i64 = sqlx::Row::try_get(&row, 0).expect("failed");
859
860 assert_eq!(failed_1, 1);
861 assert_eq!(failed_2, 1);
862 }
863
864 #[tokio::test]
865 async fn poll_database_breaks_batch_on_consume_fail() {
866 let pool = sqlite_pool().await;
867 seed_consumer_table(&pool).await;
868
869 let mut config = SqlEndpointConfig::from_uri(
870 "sql:select id, processed, failed from jobs where processed = 0 order by id?db_url=sqlite::memory:&onConsumeFailed=update jobs set failed=1 where id=:#id&breakBatchOnConsumeFail=true&initialDelay=0&delay=1",
871 )
872 .unwrap();
873 config.resolve_defaults();
874
875 let consumer = SqlConsumer::new(config.clone(), Arc::new(OnceCell::new()), None, test_rt());
876 let template = parse_query_template(&config.query, config.placeholder).unwrap();
877
878 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
879 tokio::spawn(async move {
880 loop {
881 match timeout(Duration::from_secs(2), rx.recv()).await {
882 Ok(Some(env)) => {
883 if let Some(reply_tx) = env.reply_tx {
884 let _ = reply_tx
885 .send(Err(CamelError::ProcessorError("downstream boom".into())));
886 }
887 }
888 Ok(None) => break, Err(_) => break, }
891 }
892 });
893 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
894
895 let err = consumer
896 .poll_database(&pool, &ctx, &template)
897 .await
898 .expect_err("must stop on first downstream failure");
899 assert!(err.to_string().contains("downstream boom"));
900
901 let row = sqlx::query("select failed from jobs where id = 1")
902 .fetch_one(&pool)
903 .await
904 .expect("row 1");
905 let failed_1: i64 = sqlx::Row::try_get(&row, 0).expect("failed");
906
907 let row = sqlx::query("select failed from jobs where id = 2")
908 .fetch_one(&pool)
909 .await
910 .expect("row 2");
911 let failed_2: i64 = sqlx::Row::try_get(&row, 0).expect("failed");
912
913 assert_eq!(failed_1, 1);
914 assert_eq!(failed_2, 0, "second row must not be processed");
915 }
916
917 #[tokio::test]
923 async fn consumer_no_panic_without_prior_resolve_defaults() {
924 let config = SqlEndpointConfig::from_uri(
925 "sql:select 1?db_url=sqlite::memory:&initialDelay=0&delay=1",
926 )
927 .unwrap();
928 assert!(config.max_connections.is_none());
930
931 let mut consumer = SqlConsumer::new(
932 config,
933 Arc::new(OnceCell::new()),
934 None,
935 std::sync::Arc::new(camel_component_api::test_support::NoopRuntimeObservability),
938 );
939 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
940 tokio::spawn(async move {
941 loop {
942 match timeout(Duration::from_secs(2), rx.recv()).await {
943 Ok(Some(env)) => {
944 if let Some(reply_tx) = env.reply_tx {
945 let _ = reply_tx.send(Ok(env.exchange));
946 }
947 }
948 Ok(None) => break, Err(_) => break, }
951 }
952 });
953 let token = CancellationToken::new();
954 let ctx = ConsumerContext::new(tx, token.clone(), "sql-test-route".to_string());
955
956 let consumer_handle = tokio::spawn(async move { consumer.start(ctx).await });
958
959 tokio::time::sleep(Duration::from_millis(50)).await;
961 token.cancel();
962
963 let result = consumer_handle.await.expect("task should not panic");
964 let _ = result;
966 }
967
968 #[tokio::test]
970 async fn stop_closes_pool() {
971 let pool = sqlite_pool().await;
972 seed_consumer_table(&pool).await;
973
974 let mut config = SqlEndpointConfig::from_uri(
975 "sql:select id from jobs?db_url=sqlite::memory:&onConsume=update jobs set processed=1 where id=:#id&initialDelay=0&delay=1",
976 )
977 .unwrap();
978 config.resolve_defaults();
979
980 let pool_cell = Arc::new(OnceCell::new());
981 pool_cell.set(Arc::new(pool.clone())).unwrap();
982
983 let mut consumer = SqlConsumer::new(config, pool_cell, None, test_rt());
984 consumer.stop().await.expect("stop should succeed");
985
986 assert!(
988 pool.is_closed(),
989 "Pool should be closed after consumer.stop()"
990 );
991 }
992
993 #[tokio::test]
995 async fn double_stop_is_safe() {
996 let pool = sqlite_pool().await;
997 let mut config = SqlEndpointConfig::from_uri(
998 "sql:select 1?db_url=sqlite::memory:&initialDelay=0&delay=1",
999 )
1000 .unwrap();
1001 config.resolve_defaults();
1002
1003 let pool_cell = Arc::new(OnceCell::new());
1004 pool_cell.set(Arc::new(pool.clone())).unwrap();
1005
1006 let mut consumer = SqlConsumer::new(config, pool_cell, None, test_rt());
1007 consumer.stop().await.expect("first stop should succeed");
1008 consumer
1009 .stop()
1010 .await
1011 .expect("second stop should also succeed");
1012 }
1013
1014 #[tokio::test]
1016 async fn start_after_stop_rejected() {
1017 let pool = sqlite_pool().await;
1018 let mut config = SqlEndpointConfig::from_uri(
1019 "sql:select 1?db_url=sqlite::memory:&initialDelay=0&delay=1",
1020 )
1021 .unwrap();
1022 config.resolve_defaults();
1023
1024 let pool_cell = Arc::new(OnceCell::new());
1025 pool_cell.set(Arc::new(pool.clone())).unwrap();
1026
1027 let mut consumer = SqlConsumer::new(config, pool_cell, None, test_rt());
1028 consumer.stop().await.expect("stop should succeed");
1029
1030 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
1031 tokio::spawn(async move {
1032 loop {
1033 match timeout(Duration::from_secs(2), rx.recv()).await {
1034 Ok(Some(env)) => {
1035 if let Some(reply_tx) = env.reply_tx {
1036 let _ = reply_tx.send(Ok(env.exchange));
1037 }
1038 }
1039 Ok(None) => break, Err(_) => break, }
1042 }
1043 });
1044 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
1045
1046 let result = consumer.start(ctx).await;
1047 assert!(result.is_err());
1048 let err_msg = result.unwrap_err().to_string();
1049 assert!(
1050 err_msg.contains("cannot be restarted") || err_msg.contains("after stop"),
1051 "Expected restart error, got: {}",
1052 err_msg
1053 );
1054 }
1055
1056 #[tokio::test]
1058 async fn batch_mode_per_row_post_processing() {
1059 let pool = sqlite_pool().await;
1060 seed_consumer_table(&pool).await;
1061
1062 let mut config = SqlEndpointConfig::from_uri(
1063 "sql:select id, processed, failed from jobs where processed = 0 order by id?db_url=sqlite::memory:&onConsume=update jobs set processed=1 where id=:#id&useIterator=false&initialDelay=0&delay=1",
1064 )
1065 .unwrap();
1066 config.resolve_defaults();
1067
1068 let consumer = SqlConsumer::new(config.clone(), Arc::new(OnceCell::new()), None, test_rt());
1069 let template = parse_query_template(&config.query, config.placeholder).unwrap();
1070
1071 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
1072 tokio::spawn(async move {
1073 loop {
1074 match timeout(Duration::from_secs(2), rx.recv()).await {
1075 Ok(Some(env)) => {
1076 if let Some(reply_tx) = env.reply_tx {
1077 let _ = reply_tx.send(Ok(env.exchange));
1078 }
1079 }
1080 Ok(None) => break, Err(_) => break, }
1083 }
1084 });
1085 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
1086
1087 consumer
1088 .poll_database(&pool, &ctx, &template)
1089 .await
1090 .expect("poll must succeed");
1091
1092 let row = sqlx::query("select processed from jobs where id = 1")
1094 .fetch_one(&pool)
1095 .await
1096 .expect("row 1");
1097 let processed_1: i64 = sqlx::Row::try_get(&row, 0).expect("processed");
1098
1099 let row = sqlx::query("select processed from jobs where id = 2")
1100 .fetch_one(&pool)
1101 .await
1102 .expect("row 2");
1103 let processed_2: i64 = sqlx::Row::try_get(&row, 0).expect("processed");
1104
1105 assert_eq!(
1106 processed_1, 1,
1107 "row 1 should be marked processed via per-row onConsume"
1108 );
1109 assert_eq!(
1110 processed_2, 1,
1111 "row 2 should be marked processed via per-row onConsume"
1112 );
1113 }
1114
1115 #[tokio::test]
1117 async fn batch_mode_per_row_post_processing_on_failure() {
1118 let pool = sqlite_pool().await;
1119 seed_consumer_table(&pool).await;
1120
1121 let mut config = SqlEndpointConfig::from_uri(
1122 "sql:select id, processed, failed from jobs where processed = 0 order by id?db_url=sqlite::memory:&onConsumeFailed=update jobs set failed=1 where id=:#id&useIterator=false&initialDelay=0&delay=1",
1123 )
1124 .unwrap();
1125 config.resolve_defaults();
1126
1127 let consumer = SqlConsumer::new(config.clone(), Arc::new(OnceCell::new()), None, test_rt());
1128 let template = parse_query_template(&config.query, config.placeholder).unwrap();
1129
1130 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
1131 tokio::spawn(async move {
1132 loop {
1133 match timeout(Duration::from_secs(2), rx.recv()).await {
1134 Ok(Some(env)) => {
1135 if let Some(reply_tx) = env.reply_tx {
1136 let _ = reply_tx
1137 .send(Err(CamelError::ProcessorError("downstream boom".into())));
1138 }
1139 }
1140 Ok(None) => break, Err(_) => break, }
1143 }
1144 });
1145 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
1146
1147 consumer
1148 .poll_database(&pool, &ctx, &template)
1149 .await
1150 .expect("consumer should swallow downstream errors when breakBatchOnConsumeFail=false");
1151
1152 let row = sqlx::query("select failed from jobs where id = 1")
1154 .fetch_one(&pool)
1155 .await
1156 .expect("row 1");
1157 let failed_1: i64 = sqlx::Row::try_get(&row, 0).expect("failed");
1158
1159 let row = sqlx::query("select failed from jobs where id = 2")
1160 .fetch_one(&pool)
1161 .await
1162 .expect("row 2");
1163 let failed_2: i64 = sqlx::Row::try_get(&row, 0).expect("failed");
1164
1165 assert_eq!(
1166 failed_1, 1,
1167 "row 1 should be marked failed via per-row onConsumeFailed"
1168 );
1169 assert_eq!(
1170 failed_2, 1,
1171 "row 2 should be marked failed via per-row onConsumeFailed"
1172 );
1173 }
1174
1175 #[tokio::test]
1176 async fn bridge_error_handler_routes_poll_errors_to_exchange_error() {
1177 let mut config = config();
1178 config.bridge_error_handler = true;
1179 let consumer = SqlConsumer::new(config, Arc::new(OnceCell::new()), None, test_rt());
1180
1181 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(4);
1182 tokio::spawn(async move {
1183 #[allow(clippy::never_loop)]
1184 loop {
1185 match timeout(Duration::from_secs(2), rx.recv()).await {
1186 Ok(Some(env)) => {
1187 assert!(env.exchange.error.is_some(), "exchange must carry error");
1188 if let Some(reply_tx) = env.reply_tx {
1189 let _ = reply_tx.send(Ok(env.exchange));
1190 }
1191 break;
1192 }
1193 Ok(None) => break, Err(_) => break, }
1196 }
1197 });
1198
1199 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
1200 consumer
1201 .bridge_poll_error(&ctx, CamelError::ProcessorError("poll failed".into()))
1202 .await
1203 .expect("bridging should succeed");
1204 }
1205
1206 #[tracing_test::traced_test]
1210 #[tokio::test]
1211 async fn bridged_poll_failure_emits_warn_not_error() {
1212 let pool = sqlite_pool().await;
1213 let mut config = config();
1218 config.bridge_error_handler = true;
1219 config.query = "select * from nonexistent_table".to_string();
1221 config.resolve_defaults();
1222 let consumer = SqlConsumer::new(config.clone(), Arc::new(OnceCell::new()), None, test_rt());
1223 let template = parse_query_template(&config.query, config.placeholder).unwrap();
1224
1225 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(4);
1228 tokio::spawn(async move {
1229 loop {
1230 match timeout(Duration::from_secs(2), rx.recv()).await {
1231 Ok(Some(env)) => {
1232 if let Some(reply_tx) = env.reply_tx {
1233 let _ = reply_tx.send(Ok(env.exchange));
1234 }
1235 }
1236 Ok(None) => break, Err(_) => break, }
1239 }
1240 });
1241 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
1242
1243 consumer.handle_poll_result(&pool, &ctx, &template).await;
1245
1246 assert!(
1248 !logs_contain("ERROR"),
1249 "bridged poll failure must not emit ERROR (handler owns it); check captured logs for stray ERROR lines"
1250 );
1251 assert!(
1253 logs_contain("WARN"),
1254 "bridged poll failure should emit warn! for operator visibility"
1255 );
1256 }
1257
1258 #[tracing_test::traced_test]
1268 #[tokio::test]
1269 async fn unbridged_send_and_wait_failure_emits_error_loud() {
1270 let pool = sqlite_pool().await;
1271 sqlx::query("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")
1272 .execute(&pool)
1273 .await
1274 .expect("create table");
1275 sqlx::query("INSERT INTO items (id, name) VALUES (1, 'alpha')")
1276 .execute(&pool)
1277 .await
1278 .expect("seed rows");
1279
1280 let mut config = SqlEndpointConfig::from_uri(
1281 "sql:select id, name from items order by id?db_url=sqlite::memory:&outputType=StreamList&initialDelay=0&delay=1",
1282 )
1283 .unwrap();
1284 config.resolve_defaults();
1285 config.bridge_error_handler = false;
1287 let consumer = SqlConsumer::new(
1288 config.clone(),
1289 Arc::new(OnceCell::new()),
1290 None,
1291 Arc::new(RecordingRuntime::new(Arc::new(Mutex::new(Vec::new())))),
1292 );
1293 let template = parse_query_template(&config.query, config.placeholder).unwrap();
1294
1295 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
1297 tokio::spawn(async move {
1298 loop {
1299 match timeout(Duration::from_secs(2), rx.recv()).await {
1300 Ok(Some(env)) => {
1301 if let Some(reply_tx) = env.reply_tx {
1302 let _ = reply_tx.send(Err(CamelError::ProcessorError("boom".into())));
1303 }
1304 }
1305 Ok(None) => break, Err(_) => break, }
1308 }
1309 });
1310 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
1311
1312 let _ = consumer.poll_database(&pool, &ctx, &template).await;
1313
1314 assert!(
1316 logs_contain("ERROR"),
1317 "unbridged send_and_wait failure MUST emit ERROR (consumer owns the signal)"
1318 );
1319 }
1320
1321 #[tracing_test::traced_test]
1324 #[tokio::test]
1325 async fn unbridged_handle_poll_result_emits_error_loud() {
1326 let pool = sqlite_pool().await;
1327 let mut config = config();
1330 config.bridge_error_handler = false;
1331 config.query = "select * from nonexistent_table".to_string();
1332 config.resolve_defaults();
1333 let consumer = SqlConsumer::new(
1334 config.clone(),
1335 Arc::new(OnceCell::new()),
1336 None,
1337 Arc::new(RecordingRuntime::new(Arc::new(Mutex::new(Vec::new())))),
1338 );
1339 let template = parse_query_template(&config.query, config.placeholder).unwrap();
1340
1341 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(4);
1343 tokio::spawn(async move {
1344 loop {
1345 match timeout(Duration::from_secs(2), rx.recv()).await {
1346 Ok(Some(env)) => {
1347 if let Some(reply_tx) = env.reply_tx {
1348 let _ = reply_tx.send(Ok(env.exchange));
1349 }
1350 }
1351 Ok(None) => break, Err(_) => break, }
1354 }
1355 });
1356 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
1357
1358 consumer.handle_poll_result(&pool, &ctx, &template).await;
1359
1360 assert!(
1361 logs_contain("ERROR"),
1362 "unbridged handle_poll_result failure MUST emit ERROR (consumer owns signal)"
1363 );
1364 }
1365
1366 #[tokio::test]
1367 async fn stream_list_consumer_emits_ndjson_body() {
1368 let pool = sqlite_pool().await;
1369 sqlx::query("CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")
1370 .execute(&pool)
1371 .await
1372 .expect("create table");
1373 sqlx::query("INSERT INTO items (id, name) VALUES (1, 'alpha'), (2, 'beta'), (3, 'gamma')")
1374 .execute(&pool)
1375 .await
1376 .expect("seed rows");
1377
1378 let mut config = SqlEndpointConfig::from_uri(
1379 "sql:select id, name from items order by id?db_url=sqlite::memory:&outputType=StreamList&initialDelay=0&delay=1",
1380 )
1381 .unwrap();
1382 config.resolve_defaults();
1383
1384 let consumer = SqlConsumer::new(config.clone(), Arc::new(OnceCell::new()), None, test_rt());
1385 let template = parse_query_template(&config.query, config.placeholder).unwrap();
1386
1387 let (tx, rx) = mpsc::channel::<ExchangeEnvelope>(8);
1388 let (result_tx, result_rx) = tokio::sync::oneshot::channel::<Exchange>();
1389 tokio::spawn(async move {
1390 let mut rx = rx;
1391 match timeout(Duration::from_secs(2), rx.recv()).await {
1392 Ok(Some(env)) => {
1393 if let Some(reply_tx) = env.reply_tx {
1394 let _ = reply_tx.send(Ok(env.exchange.clone()));
1395 }
1396 let _ = result_tx.send(env.exchange);
1397 }
1398 Ok(None) => {}
1399 Err(_) => {}
1400 }
1401 });
1402 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
1403
1404 consumer
1405 .poll_database(&pool, &ctx, &template)
1406 .await
1407 .expect("poll must succeed");
1408
1409 let exchange = result_rx.await.expect("should have received one exchange");
1410
1411 match exchange.input.body {
1412 Body::Stream(ref stream_body) => {
1413 let stream = stream_body.stream.clone();
1414 let mut guard = stream.lock().await;
1415 let stream_opt = guard.take();
1416 assert!(stream_opt.is_some(), "stream should be present");
1417
1418 use futures::StreamExt;
1419 let mut collected = Vec::new();
1420 let mut stream = stream_opt.unwrap();
1421 while let Some(chunk) = stream.next().await {
1422 let chunk = chunk.expect("stream chunk should not error");
1423 collected.extend_from_slice(&chunk);
1424 }
1425
1426 let ndjson = String::from_utf8(collected).expect("valid utf8");
1427 let lines: Vec<&str> = ndjson.trim().lines().collect();
1428 assert_eq!(lines.len(), 3, "should have 3 NDJSON lines");
1429
1430 let row0: serde_json::Value =
1431 serde_json::from_str(lines[0]).expect("valid json line 0");
1432 assert_eq!(row0["id"], 1);
1433 assert_eq!(row0["name"], "alpha");
1434
1435 let row1: serde_json::Value =
1436 serde_json::from_str(lines[1]).expect("valid json line 1");
1437 assert_eq!(row1["id"], 2);
1438 assert_eq!(row1["name"], "beta");
1439
1440 let row2: serde_json::Value =
1441 serde_json::from_str(lines[2]).expect("valid json line 2");
1442 assert_eq!(row2["id"], 3);
1443 assert_eq!(row2["name"], "gamma");
1444 }
1445 ref other => panic!("expected Body::Stream, got {:?}", other),
1446 }
1447 }
1448
1449 #[tokio::test]
1450 async fn stream_list_consumer_empty_result_set_emits_empty_stream() {
1451 let pool = sqlite_pool().await;
1452 sqlx::query("CREATE TABLE empty_items (id INTEGER PRIMARY KEY, name TEXT)")
1453 .execute(&pool)
1454 .await
1455 .expect("create table");
1456
1457 let mut config = SqlEndpointConfig::from_uri(
1458 "sql:select id, name from empty_items?db_url=sqlite::memory:&outputType=StreamList&initialDelay=0&delay=1",
1459 )
1460 .unwrap();
1461 config.resolve_defaults();
1462
1463 let consumer = SqlConsumer::new(config.clone(), Arc::new(OnceCell::new()), None, test_rt());
1464 let template = parse_query_template(&config.query, config.placeholder).unwrap();
1465
1466 let (tx, rx) = tokio::sync::oneshot::channel();
1467 let (mpsc_tx, mut mpsc_rx) = mpsc::channel::<ExchangeEnvelope>(8);
1468 tokio::spawn(async move {
1469 #[allow(clippy::never_loop)]
1470 loop {
1471 match timeout(Duration::from_secs(2), mpsc_rx.recv()).await {
1472 Ok(Some(env)) => {
1473 if let Some(reply_tx) = env.reply_tx {
1474 let _ = reply_tx.send(Ok(env.exchange.clone()));
1475 }
1476 let _ = tx.send(env.exchange);
1477 break;
1478 }
1479 Ok(None) => break, Err(_) => break, }
1482 }
1483 });
1484 let ctx = ConsumerContext::new(
1485 mpsc_tx,
1486 CancellationToken::new(),
1487 "sql-test-route".to_string(),
1488 );
1489
1490 consumer
1491 .poll_database(&pool, &ctx, &template)
1492 .await
1493 .expect("poll must succeed");
1494
1495 let exchange = rx
1496 .await
1497 .expect("StreamList should emit exchange even for empty results");
1498
1499 match exchange.input.body {
1500 Body::Stream(ref stream_body) => {
1501 let stream = stream_body.stream.clone();
1502 let mut guard = stream.lock().await;
1503 let stream_opt = guard.take();
1504
1505 use futures::StreamExt;
1506 let mut count = 0;
1507 if let Some(mut stream) = stream_opt {
1508 while let Some(chunk) = stream.next().await {
1509 let chunk = chunk.expect("stream chunk should not error");
1510 count += chunk.len();
1511 }
1512 }
1513 assert_eq!(count, 0, "empty table should produce zero stream bytes");
1514 }
1515 ref other => panic!("expected Body::Stream, got {:?}", other),
1516 }
1517 }
1518
1519 #[tokio::test]
1520 async fn break_on_empty_stops_after_drained_table() {
1521 let pool = sqlite_pool().await;
1522 seed_consumer_table(&pool).await;
1523
1524 let mut config = SqlEndpointConfig::from_uri(
1526 "sql:select id from jobs where processed = 0 order by id?db_url=sqlite::memory:&onConsume=update jobs set processed=1 where id=:#id&initialDelay=0&delay=50&breakOnEmpty=true&repeatCount=100",
1527 )
1528 .unwrap();
1529 config.resolve_defaults();
1530
1531 let pool_cell = Arc::new(OnceCell::new());
1534 pool_cell
1535 .set(Arc::new(pool.clone()))
1536 .expect("pool cell set");
1537 let mut consumer = SqlConsumer::new(config, pool_cell, None, test_rt());
1538
1539 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
1540 let route_cancel = CancellationToken::new();
1541 let received = Arc::new(std::sync::atomic::AtomicU32::new(0));
1545 let echo_cancel = route_cancel.clone();
1546 let counter = Arc::clone(&received);
1547 tokio::spawn(async move {
1549 loop {
1550 tokio::select! {
1551 _ = echo_cancel.cancelled() => break,
1552 Some(env) = rx.recv() => {
1553 counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1554 if let Some(reply_tx) = env.reply_tx {
1555 let _ = reply_tx.send(Ok(env.exchange));
1556 }
1557 }
1558 }
1559 }
1560 });
1561
1562 let ctx = ConsumerContext::new(tx, route_cancel.clone(), "sql-test-route".to_string());
1563
1564 let start = tokio::time::Instant::now();
1566 consumer.start(ctx).await.expect("start must succeed");
1567 let elapsed = start.elapsed();
1568
1569 let n = received.load(std::sync::atomic::Ordering::Relaxed);
1572 assert_eq!(
1573 n, 2,
1574 "productive poll should emit 2 envelopes (2 seeded rows), got {}",
1575 n
1576 );
1577
1578 let count_unprocessed: i64 =
1580 sqlx::query_scalar("select count(*) from jobs where processed = 0")
1581 .fetch_one(&pool)
1582 .await
1583 .expect("count");
1584 assert_eq!(count_unprocessed, 0);
1585
1586 assert!(
1589 elapsed < std::time::Duration::from_millis(500),
1590 "consumer should have stopped on empty poll, took {:?}",
1591 elapsed
1592 );
1593
1594 assert!(
1603 elapsed >= std::time::Duration::from_millis(100),
1604 "consumer must run a second (empty) poll before breaking on break_on_empty, \
1605 took {:?} — likely broke after the productive poll without seeing the empty one",
1606 elapsed
1607 );
1608 }
1609
1610 #[tokio::test]
1614 async fn handle_poll_result_error_does_not_signal_empty() {
1615 let pool = sqlite_pool().await;
1616
1617 let mut config = SqlEndpointConfig::from_uri(
1618 "sql:select * from this_table_does_not_exist?db_url=sqlite::memory:&breakOnEmpty=true&initialDelay=0&delay=1",
1619 )
1620 .unwrap();
1621 config.resolve_defaults();
1622
1623 let pool_cell = Arc::new(OnceCell::new());
1624 pool_cell.set(Arc::new(pool.clone())).unwrap();
1625 let consumer = SqlConsumer::new(
1628 config.clone(),
1629 pool_cell,
1630 None,
1631 Arc::new(RecordingRuntime::new(Arc::new(Mutex::new(Vec::new())))),
1632 );
1633
1634 let template = parse_query_template(&config.query, config.placeholder).unwrap();
1635
1636 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
1637 tokio::spawn(async move {
1638 loop {
1639 match timeout(Duration::from_secs(2), rx.recv()).await {
1640 Ok(Some(env)) => {
1641 if let Some(reply_tx) = env.reply_tx {
1642 let _ = reply_tx.send(Ok(env.exchange));
1643 }
1644 }
1645 Ok(None) => break, Err(_) => break, }
1648 }
1649 });
1650 let ctx = ConsumerContext::new(tx, CancellationToken::new(), "sql-test-route".to_string());
1651
1652 let outcome = consumer.handle_poll_result(&pool, &ctx, &template).await;
1653 assert!(
1654 !outcome.was_empty,
1655 "poll error must NOT signal empty (was_empty should be false)"
1656 );
1657 }
1658
1659 #[derive(Debug, Default)]
1663 struct RecordingHealth {
1664 forced: Arc<Mutex<Vec<(String, String, String)>>>,
1665 }
1666
1667 impl HealthCheckRegistry for RecordingHealth {
1668 fn force_unhealthy_for_route(&self, route_id: &str, name: &str, reason: &str) {
1669 self.forced.lock().unwrap().push((
1670 route_id.to_string(),
1671 name.to_string(),
1672 reason.to_string(),
1673 ));
1674 }
1675 }
1676
1677 struct NoopMetricsForConsumer;
1678
1679 impl MetricsCollector for NoopMetricsForConsumer {
1680 fn record_exchange_duration(&self, _: &str, _: Duration) {}
1681 fn increment_errors(&self, _: &str, _: &str) {}
1682 fn increment_exchanges(&self, _: &str) {}
1683 fn set_queue_depth(&self, _: &str, _: usize) {}
1684 fn record_circuit_breaker_change(&self, _: &str, _: &str, _: &str) {}
1685 }
1686
1687 struct RecordingRuntimeWithHealth {
1688 health: Arc<RecordingHealth>,
1689 }
1690
1691 impl RuntimeObservability for RecordingRuntimeWithHealth {
1692 fn metrics(&self) -> Arc<dyn MetricsCollector> {
1693 Arc::new(NoopMetricsForConsumer)
1694 }
1695 fn health(&self) -> Arc<dyn HealthCheckRegistry> {
1696 self.health.clone()
1697 }
1698 }
1699
1700 #[tokio::test]
1703 async fn consumer_pool_init_failure_calls_force_unhealthy_for_route() {
1704 let health = Arc::new(RecordingHealth::default());
1705 let recorded_health = health.clone();
1706 let rt: Arc<dyn RuntimeObservability> = Arc::new(RecordingRuntimeWithHealth { health });
1707
1708 let mut config = SqlEndpointConfig::from_uri(
1709 "sql:select 1?db_url=postgres://nonexistent-host:5432/nonexistent_db&retryEnabled=false&initialDelay=0&delay=1",
1710 )
1711 .unwrap();
1712 config.max_connections = Some(1);
1713 config.min_connections = Some(0);
1714 config.idle_timeout_secs = Some(300);
1715 config.max_lifetime_secs = Some(1800);
1716
1717 let mut consumer = SqlConsumer::new(config, Arc::new(OnceCell::new()), None, rt);
1718
1719 let (tx, _rx) = mpsc::channel(8);
1720 let ctx = ConsumerContext::new(
1721 tx,
1722 CancellationToken::new(),
1723 "sql-consumer-test-route".to_string(),
1724 );
1725
1726 let result = consumer.start(ctx).await;
1727 assert!(result.is_err(), "pool init should fail with bad db_url");
1728
1729 let forced = recorded_health.forced.lock().unwrap();
1730 assert_eq!(
1731 forced.len(),
1732 1,
1733 "expected one force_unhealthy_for_route call"
1734 );
1735 assert_eq!(forced[0].0, "sql-consumer-test-route");
1736 assert_eq!(forced[0].1, "g:sql:consumer-pool-init");
1737 assert!(!forced[0].2.is_empty(), "reason should be non-empty");
1738 }
1739
1740 #[tokio::test]
1744 async fn retry_loop_invokes_operation_exactly_max_attempts_times() {
1745 use camel_component_api::NetworkRetryPolicy;
1746 use std::sync::Arc;
1747 use std::sync::atomic::{AtomicU32, Ordering};
1748
1749 let policy = NetworkRetryPolicy {
1750 max_attempts: 3,
1751 initial_delay: Duration::from_millis(1),
1752 max_delay: Duration::from_millis(1),
1753 multiplier: 1.0,
1754 ..NetworkRetryPolicy::default()
1755 };
1756
1757 let calls = Arc::new(AtomicU32::new(0));
1758 let calls_clone = Arc::clone(&calls);
1759
1760 let mut attempt: u32 = 0;
1761 let _result: Result<(), ()> = tokio::time::timeout(Duration::from_secs(30), async {
1762 loop {
1763 attempt += 1;
1764 calls_clone.fetch_add(1, Ordering::SeqCst);
1765 let op_result: Result<(), ()> = Err(());
1766 match op_result {
1767 Ok(v) => break Ok(v),
1768 Err(_) if policy.should_retry(attempt) => {
1769 let delay = policy.delay_for(attempt - 1);
1770 tokio::time::sleep(delay).await;
1771 continue;
1772 }
1773 Err(_) => break Err(()),
1774 }
1775 }
1776 })
1777 .await
1778 .expect("retry loop must finish within 30s");
1779
1780 assert_eq!(
1781 calls.load(Ordering::SeqCst),
1782 3,
1783 "max_attempts=3 must yield exactly 3 invocations"
1784 );
1785 }
1786
1787 #[tokio::test]
1790 #[tracing_test::traced_test]
1791 async fn break_on_empty_ignored_in_streamlist() {
1792 let pool = sqlite_pool().await;
1793 seed_consumer_table(&pool).await;
1794
1795 let mut config = SqlEndpointConfig::from_uri(
1799 "sql:select id from jobs?db_url=sqlite::memory:&outputType=StreamList&initialDelay=0&delay=1&breakOnEmpty=true&repeatCount=3",
1800 )
1801 .unwrap();
1802 config.resolve_defaults();
1803 assert_eq!(config.output_type, SqlOutputType::StreamList);
1804 assert!(config.break_on_empty);
1805
1806 let pool_cell = Arc::new(OnceCell::new());
1807 pool_cell
1808 .set(Arc::new(pool.clone()))
1809 .expect("pool cell set");
1810 let mut consumer = SqlConsumer::new(config, pool_cell, None, test_rt());
1811
1812 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
1813 let route_cancel = CancellationToken::new();
1814 let received = Arc::new(std::sync::atomic::AtomicU32::new(0));
1815 let echo_cancel = route_cancel.clone();
1816 let counter = Arc::clone(&received);
1817 tokio::spawn(async move {
1818 loop {
1819 tokio::select! {
1820 _ = echo_cancel.cancelled() => break,
1821 Some(env) = rx.recv() => {
1822 counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1823 if let Some(reply_tx) = env.reply_tx {
1824 let _ = reply_tx.send(Ok(env.exchange));
1825 }
1826 }
1827 }
1828 }
1829 });
1830
1831 let ctx = ConsumerContext::new(tx, route_cancel, "sql-test-route".to_string());
1832 consumer.start(ctx).await.expect("start must succeed");
1833
1834 assert!(
1837 logs_contain("breakOnEmpty"),
1838 "expected StreamList warn naming breakOnEmpty"
1839 );
1840
1841 assert!(
1845 received.load(std::sync::atomic::Ordering::Relaxed) >= 2,
1846 "StreamList must not break early with breakOnEmpty, got {} exchanges",
1847 received.load(std::sync::atomic::Ordering::Relaxed)
1848 );
1849 }
1850
1851 #[tokio::test]
1852 async fn break_on_empty_false_default_loops_on_empty() {
1853 let pool = sqlite_pool().await;
1854 seed_consumer_table(&pool).await;
1856 sqlx::query("delete from jobs")
1857 .execute(&pool)
1858 .await
1859 .expect("drain");
1860
1861 let mut config = SqlEndpointConfig::from_uri(
1864 "sql:select id from jobs where processed = 0?db_url=sqlite::memory:&initialDelay=0&delay=20&repeatCount=3",
1865 )
1866 .unwrap();
1867 config.resolve_defaults();
1868 assert!(!config.break_on_empty);
1869
1870 let pool_cell = Arc::new(OnceCell::new());
1871 pool_cell
1872 .set(Arc::new(pool.clone()))
1873 .expect("pool cell set");
1874 let mut consumer = SqlConsumer::new(config, pool_cell, None, test_rt());
1875
1876 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
1877 let route_cancel = CancellationToken::new();
1878 let echo_cancel = route_cancel.clone();
1879 tokio::spawn(async move {
1880 loop {
1881 tokio::select! {
1882 _ = echo_cancel.cancelled() => break,
1883 Some(env) = rx.recv() => {
1884 if let Some(reply_tx) = env.reply_tx {
1885 let _ = reply_tx.send(Ok(env.exchange));
1886 }
1887 }
1888 }
1889 }
1890 });
1891
1892 let ctx = ConsumerContext::new(tx, route_cancel, "sql-test-route".to_string());
1893 let start = tokio::time::Instant::now();
1894 consumer.start(ctx).await.expect("start must succeed");
1895 let elapsed = start.elapsed();
1896
1897 assert!(
1901 elapsed >= std::time::Duration::from_millis(55),
1902 "consumer should run all 3 polls (breakOnEmpty=false), took {:?}",
1903 elapsed
1904 );
1905 }
1906
1907 #[tokio::test]
1908 async fn break_on_empty_with_route_empty_result_set() {
1909 let pool = sqlite_pool().await;
1910 seed_consumer_table(&pool).await;
1911 sqlx::query("delete from jobs")
1912 .execute(&pool)
1913 .await
1914 .expect("drain");
1915 sqlx::query("CREATE TABLE batch_marks (n INTEGER NOT NULL DEFAULT 0)")
1921 .execute(&pool)
1922 .await
1923 .expect("create batch_marks");
1924 sqlx::query("INSERT INTO batch_marks (n) VALUES (0)")
1925 .execute(&pool)
1926 .await
1927 .expect("seed batch_marks");
1928
1929 let mut config = SqlEndpointConfig::from_uri(
1934 "sql:select id from jobs where processed = 0?db_url=sqlite::memory:&routeEmptyResultSet=true&breakOnEmpty=true&useIterator=false&onConsumeBatchComplete=update batch_marks set n = n + 1&initialDelay=0&delay=10&repeatCount=100",
1935 )
1936 .unwrap();
1937 config.resolve_defaults();
1938
1939 let pool_cell = Arc::new(OnceCell::new());
1940 pool_cell
1941 .set(Arc::new(pool.clone()))
1942 .expect("pool cell set");
1943 let mut consumer = SqlConsumer::new(config, pool_cell, None, test_rt());
1944
1945 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
1946 let route_cancel = CancellationToken::new();
1947 let received = Arc::new(std::sync::atomic::AtomicU32::new(0));
1948 let echo_cancel = route_cancel.clone();
1949 let counter = Arc::clone(&received);
1950 tokio::spawn(async move {
1951 loop {
1952 tokio::select! {
1953 _ = echo_cancel.cancelled() => break,
1954 Some(env) = rx.recv() => {
1955 counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1956 if let Some(reply_tx) = env.reply_tx {
1957 let _ = reply_tx.send(Ok(env.exchange));
1958 }
1959 }
1960 }
1961 }
1962 });
1963
1964 let ctx = ConsumerContext::new(tx, route_cancel, "sql-test-route".to_string());
1965 let start = tokio::time::Instant::now();
1966 consumer.start(ctx).await.expect("start must succeed");
1967
1968 let n = received.load(std::sync::atomic::Ordering::Relaxed);
1972 assert_eq!(n, 1, "expected exactly 1 empty-result exchange, got {}", n);
1973 assert!(
1974 start.elapsed() < std::time::Duration::from_millis(500),
1975 "consumer should have stopped after the first empty poll, took {:?}",
1976 start.elapsed()
1977 );
1978
1979 let batch_fires: i64 = sqlx::query_scalar("select n from batch_marks")
1984 .fetch_one(&pool)
1985 .await
1986 .expect("batch_marks n");
1987 assert_eq!(
1988 batch_fires, 1,
1989 "onConsumeBatchComplete must fire exactly once on the empty-poll \
1990 fall-through before break_on_empty, got {}",
1991 batch_fires
1992 );
1993 }
1994
1995 #[tokio::test]
1996 async fn repeat_count_zero_polls_never() {
1997 let pool = sqlite_pool().await;
1998 seed_consumer_table(&pool).await;
1999
2000 let mut config = SqlEndpointConfig::from_uri(
2002 "sql:select id from jobs?db_url=sqlite::memory:&initialDelay=0&delay=1&repeatCount=0&onConsume=update jobs set processed=1 where id=:#id",
2003 )
2004 .unwrap();
2005 config.resolve_defaults();
2006
2007 let pool_cell = Arc::new(OnceCell::new());
2010 pool_cell
2011 .set(Arc::new(pool.clone()))
2012 .expect("pool cell set");
2013 let mut consumer = SqlConsumer::new(config, pool_cell, None, test_rt());
2014
2015 let (tx, mut rx) = mpsc::channel::<ExchangeEnvelope>(8);
2016 let route_cancel = CancellationToken::new();
2017 let echo_cancel = route_cancel.clone();
2018 tokio::spawn(async move {
2019 loop {
2020 tokio::select! {
2021 _ = echo_cancel.cancelled() => break,
2022 Some(env) = rx.recv() => {
2023 if let Some(reply_tx) = env.reply_tx {
2024 let _ = reply_tx.send(Ok(env.exchange));
2025 }
2026 }
2027 }
2028 }
2029 });
2030
2031 let ctx = ConsumerContext::new(tx, route_cancel, "sql-test-route".to_string());
2032 consumer.start(ctx).await.expect("start must succeed");
2033
2034 let count_processed: i64 =
2036 sqlx::query_scalar("select count(*) from jobs where processed = 1")
2037 .fetch_one(&pool)
2038 .await
2039 .expect("count");
2040 assert_eq!(count_processed, 0, "repeatCount=0 must not poll");
2041 }
2042}