1use std::pin::Pin;
2use std::sync::Arc;
3
4use futures::Stream;
5
6use crate::body::{Body, body_type_name};
7use crate::error::CamelError;
8use crate::exchange::Exchange;
9use crate::message::Message;
10
11pub type SplitExpression =
17 Arc<dyn Fn(&Exchange) -> Result<Vec<Exchange>, CamelError> + Send + Sync>;
18
19pub type StreamingSplitExpression = Arc<
26 dyn Fn(Exchange) -> Pin<Box<dyn Stream<Item = Result<Exchange, CamelError>> + Send>>
27 + Send
28 + Sync,
29>;
30
31pub fn streaming_split_type_error(body: &Body) -> CamelError {
35 CamelError::TypeConversionFailed(format!(
36 "streaming split requires body type stream, got {}; add an unmarshal step before split",
37 body_type_name(body)
38 ))
39}
40
41#[derive(Clone, Default)]
43#[non_exhaustive]
44pub enum AggregationStrategy {
45 #[default]
47 LastWins,
48 CollectAll,
50 Original,
52 Custom(Arc<dyn Fn(Exchange, Exchange) -> Exchange + Send + Sync>),
54}
55
56impl std::fmt::Debug for AggregationStrategy {
57 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
58 match self {
59 AggregationStrategy::LastWins => f.write_str("LastWins"),
60 AggregationStrategy::CollectAll => f.write_str("CollectAll"),
61 AggregationStrategy::Original => f.write_str("Original"),
62 AggregationStrategy::Custom(_) => f.write_str("Custom(..)"),
63 }
64 }
65}
66
67#[derive(
69 Clone,
70 Debug,
71 Default,
72 PartialEq,
73 Eq,
74 serde::Serialize,
75 serde::Deserialize,
76 schemars::JsonSchema,
77 ts_rs::TS,
78)]
79#[serde(rename_all = "snake_case")]
80#[ts(rename_all = "snake_case")]
81#[non_exhaustive]
82pub enum StreamSplitFormat {
83 #[default]
85 Auto,
86 Ndjson,
88 Lines,
90 Chunks,
92 Zip,
94 Tar,
96 #[serde(rename = "tar.gz")]
98 #[ts(rename = "tar.gz")]
99 TarGz,
100}
101
102#[derive(
107 Clone,
108 Debug,
109 PartialEq,
110 Eq,
111 serde::Serialize,
112 serde::Deserialize,
113 schemars::JsonSchema,
114 ts_rs::TS,
115)]
116#[serde(rename_all = "snake_case")]
117#[ts(rename_all = "snake_case")]
118pub struct StreamSplitConfig {
119 pub format: StreamSplitFormat,
121 pub max_record_bytes: usize,
123 pub batch_size: usize,
125 pub chunk_size: Option<usize>,
127 pub include_origin: bool,
129}
130
131impl Default for StreamSplitConfig {
132 fn default() -> Self {
133 Self {
134 format: StreamSplitFormat::Auto,
135 max_record_bytes: 1024 * 1024,
136 batch_size: 1,
137 chunk_size: None,
138 include_origin: true,
139 }
140 }
141}
142
143impl StreamSplitConfig {
144 pub fn validate(&self) -> Result<(), CamelError> {
157 if self.batch_size == 0 {
158 return Err(CamelError::Config(
159 "stream split batch_size must be > 0".into(),
160 ));
161 }
162 if self.max_record_bytes == 0 {
163 return Err(CamelError::Config(
164 "stream split max_record_bytes must be > 0".into(),
165 ));
166 }
167 if self.format == StreamSplitFormat::Chunks && self.chunk_size.is_none() {
168 return Err(CamelError::Config(
169 "stream split format=Chunks requires chunk_size".into(),
170 ));
171 }
172 if self.format == StreamSplitFormat::Zip && self.chunk_size.is_some() {
177 return Err(CamelError::Config(
178 "stream split format=Zip does not support chunk_size".into(),
179 ));
180 }
181 if matches!(
182 self.format,
183 StreamSplitFormat::Tar | StreamSplitFormat::TarGz
184 ) && self.chunk_size.is_some()
185 {
186 return Err(CamelError::Config(format!(
187 "stream split format={:?} is a materialized archive format and does not support chunk_size",
188 self.format
189 )));
190 }
191 if let Some(cs) = self.chunk_size
192 && cs == 0
193 {
194 return Err(CamelError::Config(
195 "stream split chunk_size must be > 0".into(),
196 ));
197 }
198 if self.format == StreamSplitFormat::Chunks
199 && let Some(cs) = self.chunk_size
200 && cs > self.max_record_bytes
201 {
202 return Err(CamelError::Config(
203 "stream split chunk_size must be <= max_record_bytes".into(),
204 ));
205 }
206 Ok(())
207 }
208}
209
210#[derive(Clone)]
212pub struct SplitterConfig {
213 pub expression: SplitExpression,
215 pub aggregation: AggregationStrategy,
217 pub parallel: bool,
219 pub parallel_limit: Option<usize>,
221 pub stop_on_exception: bool,
227 pub max_fragments: usize,
233}
234
235impl std::fmt::Debug for SplitterConfig {
236 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
237 f.debug_struct("SplitterConfig")
238 .field("expression", &"<split-expression>")
239 .field("aggregation", &self.aggregation)
240 .field("parallel", &self.parallel)
241 .field("parallel_limit", &self.parallel_limit)
242 .field("stop_on_exception", &self.stop_on_exception)
243 .field("max_fragments", &self.max_fragments)
244 .finish()
245 }
246}
247
248impl SplitterConfig {
249 pub fn new(expression: SplitExpression) -> Self {
251 Self {
252 expression,
253 aggregation: AggregationStrategy::default(),
254 parallel: false,
255 parallel_limit: None,
256 stop_on_exception: true,
257 max_fragments: 100_000,
258 }
259 }
260
261 pub fn aggregation(mut self, strategy: AggregationStrategy) -> Self {
263 self.aggregation = strategy;
264 self
265 }
266
267 pub fn parallel(mut self, parallel: bool) -> Self {
269 self.parallel = parallel;
270 self
271 }
272
273 pub fn parallel_limit(mut self, limit: usize) -> Self {
275 self.parallel_limit = Some(limit);
276 self
277 }
278
279 pub fn stop_on_exception(mut self, stop: bool) -> Self {
284 self.stop_on_exception = stop;
285 self
286 }
287
288 pub fn max_fragments(mut self, max: usize) -> Self {
290 self.max_fragments = max;
291 self
292 }
293
294 pub fn validate(&self) -> Result<(), CamelError> {
299 if self.parallel && self.parallel_limit == Some(0) {
300 return Err(CamelError::Config(
301 "splitter parallel_limit must be > 0".to_string(),
302 ));
303 }
304 if self.max_fragments == 0 {
305 return Err(CamelError::Config(
306 "splitter max_fragments must be > 0".to_string(),
307 ));
308 }
309 Ok(())
310 }
311}
312
313pub fn fragment_exchange(parent: &Exchange, body: Body) -> Exchange {
338 let mut msg = Message::new(body);
339 msg.headers = parent.input.headers.clone();
340 let mut ex = Exchange::new(msg);
341 ex.properties = parent.properties.clone();
342 ex.pattern = parent.pattern;
343 ex.otel_context = parent.otel_context.clone();
345 ex
346}
347
348pub fn split_body_lines() -> SplitExpression {
354 Arc::new(|exchange: &Exchange| {
355 let text = match &exchange.input.body {
356 Body::Text(s) => s.as_str(),
357 Body::Empty => return Ok(Vec::new()),
358 _ => {
359 return Err(CamelError::TypeConversionFailed(format!(
360 "split expression 'body_lines' requires body type text, got {received}; add an unmarshal step before split",
361 received = body_type_name(&exchange.input.body)
362 )));
363 }
364 };
365 Ok(text
366 .lines()
367 .map(|line| fragment_exchange(exchange, Body::Text(line.to_string())))
368 .collect())
369 })
370}
371
372pub fn split_body_json_array() -> SplitExpression {
383 Arc::new(|exchange: &Exchange| {
384 let arr = match &exchange.input.body {
385 Body::Json(serde_json::Value::Array(arr)) => arr,
386 Body::Empty => return Ok(Vec::new()),
387 Body::Json(_) => {
388 return Err(CamelError::TypeConversionFailed(
389 "split expression 'body_json_array' requires body type json (array), got json (non-array); add an unmarshal step before split"
390 .to_string(),
391 ))
392 }
393 _ => {
394 return Err(CamelError::TypeConversionFailed(format!(
395 "split expression 'body_json_array' requires body type json (array), got {received}; add an unmarshal step before split",
396 received = body_type_name(&exchange.input.body)
397 )))
398 }
399 };
400 Ok(arr
401 .iter()
402 .map(|val| match val {
403 serde_json::Value::String(s) => fragment_exchange(exchange, Body::Text(s.clone())),
404 other => fragment_exchange(exchange, Body::Json(other.clone())),
405 })
406 .collect())
407 })
408}
409
410pub fn split_body<F>(f: F) -> SplitExpression
415where
416 F: Fn(&Body) -> Vec<Body> + Send + Sync + 'static,
417{
418 Arc::new(move |exchange: &Exchange| {
419 Ok(f(&exchange.input.body)
420 .into_iter()
421 .map(|body| fragment_exchange(exchange, body))
422 .collect())
423 })
424}
425
426#[cfg(test)]
427mod tests {
428 use super::*;
429 use crate::value::Value;
430
431 #[test]
432 fn test_split_body_lines() {
433 let mut ex = Exchange::new(Message::new("a\nb\nc"));
434 ex.input.set_header("source", Value::String("test".into()));
435 ex.set_property("trace", Value::Bool(true));
436
437 let fragments = split_body_lines()(&ex).unwrap();
438 assert_eq!(fragments.len(), 3);
439 assert_eq!(fragments[0].input.body.as_text(), Some("a"));
440 assert_eq!(fragments[1].input.body.as_text(), Some("b"));
441 assert_eq!(fragments[2].input.body.as_text(), Some("c"));
442
443 for frag in &fragments {
445 assert_eq!(
446 frag.input.header("source"),
447 Some(&Value::String("test".into()))
448 );
449 assert_eq!(frag.property("trace"), Some(&Value::Bool(true)));
450 }
451 }
452
453 #[test]
454 fn test_split_body_lines_empty() {
455 let ex = Exchange::new(Message::default()); let fragments = split_body_lines()(&ex).unwrap();
457 assert!(fragments.is_empty());
458 }
459
460 #[test]
461 fn test_split_body_json_array() {
462 let arr = serde_json::json!([1, 2, 3]);
463 let ex = Exchange::new(Message::new(arr));
464
465 let fragments = split_body_json_array()(&ex).unwrap();
466 assert_eq!(fragments.len(), 3);
467 assert!(matches!(&fragments[0].input.body, Body::Json(v) if *v == serde_json::json!(1)));
468 assert!(matches!(&fragments[1].input.body, Body::Json(v) if *v == serde_json::json!(2)));
469 assert!(matches!(&fragments[2].input.body, Body::Json(v) if *v == serde_json::json!(3)));
470 }
471
472 #[test]
473 fn split_body_json_array_string_elements_become_text() {
474 let ex = Exchange::new(Message::new(serde_json::json!(["", "a", "b"])));
478
479 let fragments = split_body_json_array()(&ex).unwrap();
480 assert_eq!(fragments.len(), 3);
481 assert!(
482 matches!(&fragments[0].input.body, Body::Text(s) if s.is_empty()),
483 "fragment 0 must be Body::Text(\"\"), got {:?}",
484 fragments[0].input.body
485 );
486 assert!(matches!(&fragments[1].input.body, Body::Text(s) if s == "a"));
487 assert!(matches!(&fragments[2].input.body, Body::Text(s) if s == "b"));
488 for frag in &fragments {
491 let text = match &frag.input.body {
492 Body::Text(s) => s.as_str(),
493 other => panic!("expected Body::Text fragment, got {other:?}"),
494 };
495 assert!(
496 !text.contains('"'),
497 "fragment body must not carry a quote character, got {text:?}"
498 );
499 }
500 }
501
502 #[test]
503 fn test_split_body_json_array_non_string_elements_stay_json() {
504 let ex = Exchange::new(Message::new(serde_json::json!([1, {"k": "v"}, null])));
505
506 let fragments = split_body_json_array()(&ex).unwrap();
507 assert_eq!(fragments.len(), 3);
508 assert!(matches!(&fragments[0].input.body, Body::Json(v) if *v == serde_json::json!(1)));
509 assert!(matches!(&fragments[1].input.body, Body::Json(v)
510 if *v == serde_json::json!({"k": "v"})));
511 assert!(matches!(&fragments[2].input.body, Body::Json(v) if v.is_null()));
512 }
513
514 #[test]
515 fn test_split_body_json_array_not_array() {
516 let obj = serde_json::json!({"not": "array"});
517 let ex = Exchange::new(Message::new(obj));
518
519 let err = split_body_json_array()(&ex).unwrap_err();
520 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
521 assert!(err.to_string().contains("json (non-array)"));
522 }
523
524 #[test]
525 fn test_split_body_lines_wrong_type_json_errors() {
526 let ex = Exchange::new(Message::new(serde_json::json!({"a": 1})));
527
528 let err = split_body_lines()(&ex).unwrap_err();
529 let msg = err.to_string();
530 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
531 for needle in [
532 "body_lines",
533 "json",
534 "text",
535 "add an unmarshal step before split",
536 ] {
537 assert!(msg.contains(needle), "message '{msg}' missing '{needle}'");
538 }
539 }
540
541 #[test]
542 fn test_split_body_json_array_wrong_type_text_errors() {
543 let ex = Exchange::new(Message::new("x"));
544
545 let err = split_body_json_array()(&ex).unwrap_err();
546 let msg = err.to_string();
547 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
548 for needle in [
549 "body_json_array",
550 "text",
551 "json (array)",
552 "add an unmarshal step before split",
553 ] {
554 assert!(msg.contains(needle), "message '{msg}' missing '{needle}'");
555 }
556 }
557
558 #[test]
559 fn test_split_body_json_array_non_array_json_errors() {
560 let ex = Exchange::new(Message::new(serde_json::json!({"o": 1})));
561
562 let err = split_body_json_array()(&ex).unwrap_err();
563 let msg = err.to_string();
564 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
565 assert!(msg.contains("json (non-array)"));
566 }
567
568 #[test]
569 fn test_split_body_lines_empty_body_ok() {
570 let ex = Exchange::new(Message::default()); let fragments = split_body_lines()(&ex).unwrap();
572 assert!(fragments.is_empty());
573 }
574
575 #[test]
576 fn test_split_body_json_array_empty_body_ok() {
577 let ex = Exchange::new(Message::default()); let fragments = split_body_json_array()(&ex).unwrap();
579 assert!(fragments.is_empty());
580 }
581
582 #[test]
583 fn test_split_body_json_array_empty_array_ok() {
584 let ex = Exchange::new(Message::new(serde_json::json!([])));
585 let fragments = split_body_json_array()(&ex).unwrap();
586 assert!(fragments.is_empty());
587 }
588
589 #[test]
590 fn test_split_body_lines_empty_text_ok() {
591 let ex = Exchange::new(Message::new(""));
592 let fragments = split_body_lines()(&ex).unwrap();
593 assert!(fragments.is_empty());
594 }
595
596 #[test]
597 fn test_split_error_omits_payload() {
598 let ex = Exchange::new(Message::new(serde_json::json!({
599 "secret": "SECRET-8f31a"
600 })));
601
602 let err = split_body_lines()(&ex).unwrap_err();
603 let msg = err.to_string();
604 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
605 for needle in [
606 "body_lines",
607 "json",
608 "text",
609 "add an unmarshal step before split",
610 ] {
611 assert!(msg.contains(needle), "message '{msg}' missing '{needle}'");
612 }
613 assert!(
614 !msg.contains("SECRET-8f31a"),
615 "message '{msg}' leaks payload"
616 );
617 }
618
619 #[test]
620 fn test_split_body_custom() {
621 let splitter = split_body(|body: &Body| match body {
622 Body::Text(s) => s
623 .split(',')
624 .map(|part| Body::Text(part.trim().to_string()))
625 .collect(),
626 _ => Vec::new(),
627 });
628
629 let mut ex = Exchange::new(Message::new("x, y, z"));
630 ex.set_property("id", Value::from(42));
631
632 let fragments = splitter(&ex).unwrap();
633 assert_eq!(fragments.len(), 3);
634 assert_eq!(fragments[0].input.body.as_text(), Some("x"));
635 assert_eq!(fragments[1].input.body.as_text(), Some("y"));
636 assert_eq!(fragments[2].input.body.as_text(), Some("z"));
637
638 for frag in &fragments {
640 assert_eq!(frag.property("id"), Some(&Value::from(42)));
641 }
642 }
643
644 #[test]
645 fn test_splitter_config_defaults() {
646 let config = SplitterConfig::new(split_body_lines());
647 assert!(matches!(config.aggregation, AggregationStrategy::LastWins));
648 assert!(!config.parallel);
649 assert!(config.parallel_limit.is_none());
650 assert!(config.stop_on_exception);
651 }
652
653 #[test]
654 fn test_splitter_config_builder() {
655 let config = SplitterConfig::new(split_body_lines())
656 .aggregation(AggregationStrategy::CollectAll)
657 .parallel(true)
658 .parallel_limit(4)
659 .stop_on_exception(false);
660
661 assert!(matches!(
662 config.aggregation,
663 AggregationStrategy::CollectAll
664 ));
665 assert!(config.parallel);
666 assert_eq!(config.parallel_limit, Some(4));
667 assert!(!config.stop_on_exception);
668 }
669
670 #[test]
671 fn test_splitter_config_default_max_fragments() {
672 let cfg = SplitterConfig::new(Arc::new(|_: &Exchange| Ok(Vec::new())) as SplitExpression);
673 assert_eq!(cfg.max_fragments, 100_000);
674 }
675
676 #[test]
677 fn test_splitter_config_rejects_zero_max_fragments() {
678 let cfg = SplitterConfig::new(Arc::new(|_: &Exchange| Ok(Vec::new())) as SplitExpression)
679 .max_fragments(0);
680 assert!(cfg.validate().is_err());
681 }
682
683 #[test]
684 fn test_fragment_exchange_inherits_otel_context() {
685 use opentelemetry::Context;
686 use opentelemetry::trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId};
687
688 let mut parent = Exchange::new(Message::new("test"));
690 let trace_id = TraceId::from_bytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123]);
691 let span_id = SpanId::from_bytes([0, 0, 0, 0, 0, 0, 1, 200]);
692 let span_context = SpanContext::new(
693 trace_id,
694 span_id,
695 TraceFlags::SAMPLED,
696 true,
697 Default::default(),
698 );
699 let expected_trace_id = span_context.trace_id();
700 parent.otel_context = Context::current().with_remote_span_context(span_context);
701
702 let fragments = split_body_lines()(&parent).unwrap();
704 assert!(!fragments.is_empty(), "Should have at least one fragment");
705
706 for fragment in &fragments {
708 let span = fragment.otel_context.span();
709 let frag_span_ctx = span.span_context();
710 assert!(
711 frag_span_ctx.is_valid(),
712 "Fragment should have valid span context"
713 );
714 assert_eq!(
715 frag_span_ctx.trace_id(),
716 expected_trace_id,
717 "Fragment should have same trace ID as parent"
718 );
719 }
720 }
721
722 #[test]
723 fn test_stream_split_config_defaults_valid() {
724 let config = StreamSplitConfig::default();
725 assert!(config.validate().is_ok());
726 }
727
728 #[test]
729 fn test_stream_split_config_batch_size_zero_rejected() {
730 let config = StreamSplitConfig {
731 batch_size: 0,
732 ..Default::default()
733 };
734 let err = config.validate().unwrap_err();
735 assert!(err.to_string().contains("batch_size"));
736 }
737
738 #[test]
739 fn test_stream_split_config_max_record_bytes_zero_rejected() {
740 let config = StreamSplitConfig {
741 max_record_bytes: 0,
742 ..Default::default()
743 };
744 let err = config.validate().unwrap_err();
745 assert!(err.to_string().contains("max_record_bytes"));
746 }
747
748 #[test]
749 fn test_stream_split_config_chunks_requires_chunk_size() {
750 let config = StreamSplitConfig {
751 format: StreamSplitFormat::Chunks,
752 chunk_size: None,
753 ..Default::default()
754 };
755 let err = config.validate().unwrap_err();
756 assert!(err.to_string().contains("Chunks requires chunk_size"));
757 }
758
759 #[test]
760 fn test_stream_split_config_chunk_size_zero_rejected() {
761 let config = StreamSplitConfig {
762 format: StreamSplitFormat::Chunks,
763 chunk_size: Some(0),
764 ..Default::default()
765 };
766 let err = config.validate().unwrap_err();
767 assert!(err.to_string().contains("chunk_size must be > 0"));
768 }
769
770 #[test]
771 fn test_stream_split_config_chunk_size_exceeds_max_record_bytes() {
772 let config = StreamSplitConfig {
773 format: StreamSplitFormat::Chunks,
774 chunk_size: Some(2000),
775 max_record_bytes: 1000,
776 ..Default::default()
777 };
778 let err = config.validate().unwrap_err();
779 assert!(
780 err.to_string()
781 .contains("chunk_size must be <= max_record_bytes")
782 );
783 }
784
785 #[test]
786 fn test_stream_split_config_zip_rejects_chunk_size() {
787 let config = StreamSplitConfig {
788 format: StreamSplitFormat::Zip,
789 chunk_size: Some(1024),
790 ..Default::default()
791 };
792 let err = config.validate().unwrap_err();
793 assert!(err.to_string().contains("Zip does not support chunk_size"));
794 }
795
796 #[test]
797 fn test_all_fragments_share_same_trace_context() {
798 use opentelemetry::Context;
799 use opentelemetry::trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId};
800
801 let mut parent = Exchange::new(Message::new("line1\nline2\nline3"));
803 let trace_id =
804 TraceId::from_bytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x3B, 0x9A, 0xCA, 0x09]);
805 let span_id = SpanId::from_bytes([0, 0, 0, 0, 0, 0, 0, 111]);
806 let span_context = SpanContext::new(
807 trace_id,
808 span_id,
809 TraceFlags::SAMPLED,
810 true,
811 Default::default(),
812 );
813 parent.otel_context = Context::current().with_remote_span_context(span_context);
814
815 let fragments = split_body_lines()(&parent).unwrap();
816 assert_eq!(fragments.len(), 3);
817
818 let trace_ids: Vec<_> = fragments
820 .iter()
821 .map(|f| {
822 let span = f.otel_context.span();
823 span.span_context().trace_id()
824 })
825 .collect();
826
827 assert!(
828 trace_ids.iter().all(|&id| id == trace_id),
829 "All fragments should have the same trace ID"
830 );
831 }
832}