1use std::pin::Pin;
2use std::sync::Arc;
3
4use futures::Stream;
5
6use crate::body::{Body, body_type_name};
7use crate::error::CamelError;
8use crate::exchange::Exchange;
9use crate::message::Message;
10
11pub type SplitExpression =
17 Arc<dyn Fn(&Exchange) -> Result<Vec<Exchange>, CamelError> + Send + Sync>;
18
19pub type StreamingSplitExpression = Arc<
26 dyn Fn(Exchange) -> Pin<Box<dyn Stream<Item = Result<Exchange, CamelError>> + Send>>
27 + Send
28 + Sync,
29>;
30
31pub fn streaming_split_type_error(body: &Body) -> CamelError {
35 CamelError::TypeConversionFailed(format!(
36 "streaming split requires body type stream, got {}; add an unmarshal step before split",
37 body_type_name(body)
38 ))
39}
40
41#[derive(Clone, Default)]
43#[non_exhaustive]
44pub enum AggregationStrategy {
45 #[default]
47 LastWins,
48 CollectAll,
50 Original,
52 Custom(Arc<dyn Fn(Exchange, Exchange) -> Exchange + Send + Sync>),
54}
55
56impl std::fmt::Debug for AggregationStrategy {
57 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
58 match self {
59 AggregationStrategy::LastWins => f.write_str("LastWins"),
60 AggregationStrategy::CollectAll => f.write_str("CollectAll"),
61 AggregationStrategy::Original => f.write_str("Original"),
62 AggregationStrategy::Custom(_) => f.write_str("Custom(..)"),
63 }
64 }
65}
66
67#[derive(
69 Clone,
70 Debug,
71 Default,
72 PartialEq,
73 Eq,
74 serde::Serialize,
75 serde::Deserialize,
76 schemars::JsonSchema,
77 ts_rs::TS,
78)]
79#[serde(rename_all = "snake_case")]
80#[ts(rename_all = "snake_case")]
81#[non_exhaustive]
82pub enum StreamSplitFormat {
83 #[default]
85 Auto,
86 Ndjson,
88 Lines,
90 Chunks,
92 Zip,
94}
95
96#[derive(
101 Clone,
102 Debug,
103 PartialEq,
104 Eq,
105 serde::Serialize,
106 serde::Deserialize,
107 schemars::JsonSchema,
108 ts_rs::TS,
109)]
110#[serde(rename_all = "snake_case")]
111#[ts(rename_all = "snake_case")]
112pub struct StreamSplitConfig {
113 pub format: StreamSplitFormat,
115 pub max_record_bytes: usize,
117 pub batch_size: usize,
119 pub chunk_size: Option<usize>,
121 pub include_origin: bool,
123}
124
125impl Default for StreamSplitConfig {
126 fn default() -> Self {
127 Self {
128 format: StreamSplitFormat::Auto,
129 max_record_bytes: 1024 * 1024,
130 batch_size: 1,
131 chunk_size: None,
132 include_origin: true,
133 }
134 }
135}
136
137impl StreamSplitConfig {
138 pub fn validate(&self) -> Result<(), CamelError> {
149 if self.batch_size == 0 {
150 return Err(CamelError::Config(
151 "stream split batch_size must be > 0".into(),
152 ));
153 }
154 if self.max_record_bytes == 0 {
155 return Err(CamelError::Config(
156 "stream split max_record_bytes must be > 0".into(),
157 ));
158 }
159 if self.format == StreamSplitFormat::Chunks && self.chunk_size.is_none() {
160 return Err(CamelError::Config(
161 "stream split format=Chunks requires chunk_size".into(),
162 ));
163 }
164 if self.format == StreamSplitFormat::Zip && self.chunk_size.is_some() {
167 return Err(CamelError::Config(
168 "stream split format=Zip does not support chunk_size".into(),
169 ));
170 }
171 if let Some(cs) = self.chunk_size
172 && cs == 0
173 {
174 return Err(CamelError::Config(
175 "stream split chunk_size must be > 0".into(),
176 ));
177 }
178 if self.format == StreamSplitFormat::Chunks
179 && let Some(cs) = self.chunk_size
180 && cs > self.max_record_bytes
181 {
182 return Err(CamelError::Config(
183 "stream split chunk_size must be <= max_record_bytes".into(),
184 ));
185 }
186 Ok(())
187 }
188}
189
190#[derive(Clone)]
192pub struct SplitterConfig {
193 pub expression: SplitExpression,
195 pub aggregation: AggregationStrategy,
197 pub parallel: bool,
199 pub parallel_limit: Option<usize>,
201 pub stop_on_exception: bool,
207 pub max_fragments: usize,
213}
214
215impl std::fmt::Debug for SplitterConfig {
216 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
217 f.debug_struct("SplitterConfig")
218 .field("expression", &"<split-expression>")
219 .field("aggregation", &self.aggregation)
220 .field("parallel", &self.parallel)
221 .field("parallel_limit", &self.parallel_limit)
222 .field("stop_on_exception", &self.stop_on_exception)
223 .field("max_fragments", &self.max_fragments)
224 .finish()
225 }
226}
227
228impl SplitterConfig {
229 pub fn new(expression: SplitExpression) -> Self {
231 Self {
232 expression,
233 aggregation: AggregationStrategy::default(),
234 parallel: false,
235 parallel_limit: None,
236 stop_on_exception: true,
237 max_fragments: 100_000,
238 }
239 }
240
241 pub fn aggregation(mut self, strategy: AggregationStrategy) -> Self {
243 self.aggregation = strategy;
244 self
245 }
246
247 pub fn parallel(mut self, parallel: bool) -> Self {
249 self.parallel = parallel;
250 self
251 }
252
253 pub fn parallel_limit(mut self, limit: usize) -> Self {
255 self.parallel_limit = Some(limit);
256 self
257 }
258
259 pub fn stop_on_exception(mut self, stop: bool) -> Self {
264 self.stop_on_exception = stop;
265 self
266 }
267
268 pub fn max_fragments(mut self, max: usize) -> Self {
270 self.max_fragments = max;
271 self
272 }
273
274 pub fn validate(&self) -> Result<(), CamelError> {
279 if self.parallel && self.parallel_limit == Some(0) {
280 return Err(CamelError::Config(
281 "splitter parallel_limit must be > 0".to_string(),
282 ));
283 }
284 if self.max_fragments == 0 {
285 return Err(CamelError::Config(
286 "splitter max_fragments must be > 0".to_string(),
287 ));
288 }
289 Ok(())
290 }
291}
292
293pub fn fragment_exchange(parent: &Exchange, body: Body) -> Exchange {
318 let mut msg = Message::new(body);
319 msg.headers = parent.input.headers.clone();
320 let mut ex = Exchange::new(msg);
321 ex.properties = parent.properties.clone();
322 ex.pattern = parent.pattern;
323 ex.otel_context = parent.otel_context.clone();
325 ex
326}
327
328pub fn split_body_lines() -> SplitExpression {
334 Arc::new(|exchange: &Exchange| {
335 let text = match &exchange.input.body {
336 Body::Text(s) => s.as_str(),
337 Body::Empty => return Ok(Vec::new()),
338 _ => {
339 return Err(CamelError::TypeConversionFailed(format!(
340 "split expression 'body_lines' requires body type text, got {received}; add an unmarshal step before split",
341 received = body_type_name(&exchange.input.body)
342 )));
343 }
344 };
345 Ok(text
346 .lines()
347 .map(|line| fragment_exchange(exchange, Body::Text(line.to_string())))
348 .collect())
349 })
350}
351
352pub fn split_body_json_array() -> SplitExpression {
358 Arc::new(|exchange: &Exchange| {
359 let arr = match &exchange.input.body {
360 Body::Json(serde_json::Value::Array(arr)) => arr,
361 Body::Empty => return Ok(Vec::new()),
362 Body::Json(_) => {
363 return Err(CamelError::TypeConversionFailed(
364 "split expression 'body_json_array' requires body type json (array), got json (non-array); add an unmarshal step before split"
365 .to_string(),
366 ))
367 }
368 _ => {
369 return Err(CamelError::TypeConversionFailed(format!(
370 "split expression 'body_json_array' requires body type json (array), got {received}; add an unmarshal step before split",
371 received = body_type_name(&exchange.input.body)
372 )))
373 }
374 };
375 Ok(arr
376 .iter()
377 .map(|val| fragment_exchange(exchange, Body::Json(val.clone())))
378 .collect())
379 })
380}
381
382pub fn split_body<F>(f: F) -> SplitExpression
387where
388 F: Fn(&Body) -> Vec<Body> + Send + Sync + 'static,
389{
390 Arc::new(move |exchange: &Exchange| {
391 Ok(f(&exchange.input.body)
392 .into_iter()
393 .map(|body| fragment_exchange(exchange, body))
394 .collect())
395 })
396}
397
398#[cfg(test)]
399mod tests {
400 use super::*;
401 use crate::value::Value;
402
403 #[test]
404 fn test_split_body_lines() {
405 let mut ex = Exchange::new(Message::new("a\nb\nc"));
406 ex.input.set_header("source", Value::String("test".into()));
407 ex.set_property("trace", Value::Bool(true));
408
409 let fragments = split_body_lines()(&ex).unwrap();
410 assert_eq!(fragments.len(), 3);
411 assert_eq!(fragments[0].input.body.as_text(), Some("a"));
412 assert_eq!(fragments[1].input.body.as_text(), Some("b"));
413 assert_eq!(fragments[2].input.body.as_text(), Some("c"));
414
415 for frag in &fragments {
417 assert_eq!(
418 frag.input.header("source"),
419 Some(&Value::String("test".into()))
420 );
421 assert_eq!(frag.property("trace"), Some(&Value::Bool(true)));
422 }
423 }
424
425 #[test]
426 fn test_split_body_lines_empty() {
427 let ex = Exchange::new(Message::default()); let fragments = split_body_lines()(&ex).unwrap();
429 assert!(fragments.is_empty());
430 }
431
432 #[test]
433 fn test_split_body_json_array() {
434 let arr = serde_json::json!([1, 2, 3]);
435 let ex = Exchange::new(Message::new(arr));
436
437 let fragments = split_body_json_array()(&ex).unwrap();
438 assert_eq!(fragments.len(), 3);
439 assert!(matches!(&fragments[0].input.body, Body::Json(v) if *v == serde_json::json!(1)));
440 assert!(matches!(&fragments[1].input.body, Body::Json(v) if *v == serde_json::json!(2)));
441 assert!(matches!(&fragments[2].input.body, Body::Json(v) if *v == serde_json::json!(3)));
442 }
443
444 #[test]
445 fn test_split_body_json_array_not_array() {
446 let obj = serde_json::json!({"not": "array"});
447 let ex = Exchange::new(Message::new(obj));
448
449 let err = split_body_json_array()(&ex).unwrap_err();
450 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
451 assert!(err.to_string().contains("json (non-array)"));
452 }
453
454 #[test]
455 fn test_split_body_lines_wrong_type_json_errors() {
456 let ex = Exchange::new(Message::new(serde_json::json!({"a": 1})));
457
458 let err = split_body_lines()(&ex).unwrap_err();
459 let msg = err.to_string();
460 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
461 for needle in [
462 "body_lines",
463 "json",
464 "text",
465 "add an unmarshal step before split",
466 ] {
467 assert!(msg.contains(needle), "message '{msg}' missing '{needle}'");
468 }
469 }
470
471 #[test]
472 fn test_split_body_json_array_wrong_type_text_errors() {
473 let ex = Exchange::new(Message::new("x"));
474
475 let err = split_body_json_array()(&ex).unwrap_err();
476 let msg = err.to_string();
477 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
478 for needle in [
479 "body_json_array",
480 "text",
481 "json (array)",
482 "add an unmarshal step before split",
483 ] {
484 assert!(msg.contains(needle), "message '{msg}' missing '{needle}'");
485 }
486 }
487
488 #[test]
489 fn test_split_body_json_array_non_array_json_errors() {
490 let ex = Exchange::new(Message::new(serde_json::json!({"o": 1})));
491
492 let err = split_body_json_array()(&ex).unwrap_err();
493 let msg = err.to_string();
494 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
495 assert!(msg.contains("json (non-array)"));
496 }
497
498 #[test]
499 fn test_split_body_lines_empty_body_ok() {
500 let ex = Exchange::new(Message::default()); let fragments = split_body_lines()(&ex).unwrap();
502 assert!(fragments.is_empty());
503 }
504
505 #[test]
506 fn test_split_body_json_array_empty_body_ok() {
507 let ex = Exchange::new(Message::default()); let fragments = split_body_json_array()(&ex).unwrap();
509 assert!(fragments.is_empty());
510 }
511
512 #[test]
513 fn test_split_body_json_array_empty_array_ok() {
514 let ex = Exchange::new(Message::new(serde_json::json!([])));
515 let fragments = split_body_json_array()(&ex).unwrap();
516 assert!(fragments.is_empty());
517 }
518
519 #[test]
520 fn test_split_body_lines_empty_text_ok() {
521 let ex = Exchange::new(Message::new(""));
522 let fragments = split_body_lines()(&ex).unwrap();
523 assert!(fragments.is_empty());
524 }
525
526 #[test]
527 fn test_split_error_omits_payload() {
528 let ex = Exchange::new(Message::new(serde_json::json!({
529 "secret": "SECRET-8f31a"
530 })));
531
532 let err = split_body_lines()(&ex).unwrap_err();
533 let msg = err.to_string();
534 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
535 for needle in [
536 "body_lines",
537 "json",
538 "text",
539 "add an unmarshal step before split",
540 ] {
541 assert!(msg.contains(needle), "message '{msg}' missing '{needle}'");
542 }
543 assert!(
544 !msg.contains("SECRET-8f31a"),
545 "message '{msg}' leaks payload"
546 );
547 }
548
549 #[test]
550 fn test_split_body_custom() {
551 let splitter = split_body(|body: &Body| match body {
552 Body::Text(s) => s
553 .split(',')
554 .map(|part| Body::Text(part.trim().to_string()))
555 .collect(),
556 _ => Vec::new(),
557 });
558
559 let mut ex = Exchange::new(Message::new("x, y, z"));
560 ex.set_property("id", Value::from(42));
561
562 let fragments = splitter(&ex).unwrap();
563 assert_eq!(fragments.len(), 3);
564 assert_eq!(fragments[0].input.body.as_text(), Some("x"));
565 assert_eq!(fragments[1].input.body.as_text(), Some("y"));
566 assert_eq!(fragments[2].input.body.as_text(), Some("z"));
567
568 for frag in &fragments {
570 assert_eq!(frag.property("id"), Some(&Value::from(42)));
571 }
572 }
573
574 #[test]
575 fn test_splitter_config_defaults() {
576 let config = SplitterConfig::new(split_body_lines());
577 assert!(matches!(config.aggregation, AggregationStrategy::LastWins));
578 assert!(!config.parallel);
579 assert!(config.parallel_limit.is_none());
580 assert!(config.stop_on_exception);
581 }
582
583 #[test]
584 fn test_splitter_config_builder() {
585 let config = SplitterConfig::new(split_body_lines())
586 .aggregation(AggregationStrategy::CollectAll)
587 .parallel(true)
588 .parallel_limit(4)
589 .stop_on_exception(false);
590
591 assert!(matches!(
592 config.aggregation,
593 AggregationStrategy::CollectAll
594 ));
595 assert!(config.parallel);
596 assert_eq!(config.parallel_limit, Some(4));
597 assert!(!config.stop_on_exception);
598 }
599
600 #[test]
601 fn test_splitter_config_default_max_fragments() {
602 let cfg = SplitterConfig::new(Arc::new(|_: &Exchange| Ok(Vec::new())) as SplitExpression);
603 assert_eq!(cfg.max_fragments, 100_000);
604 }
605
606 #[test]
607 fn test_splitter_config_rejects_zero_max_fragments() {
608 let cfg = SplitterConfig::new(Arc::new(|_: &Exchange| Ok(Vec::new())) as SplitExpression)
609 .max_fragments(0);
610 assert!(cfg.validate().is_err());
611 }
612
613 #[test]
614 fn test_fragment_exchange_inherits_otel_context() {
615 use opentelemetry::Context;
616 use opentelemetry::trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId};
617
618 let mut parent = Exchange::new(Message::new("test"));
620 let trace_id = TraceId::from_bytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123]);
621 let span_id = SpanId::from_bytes([0, 0, 0, 0, 0, 0, 1, 200]);
622 let span_context = SpanContext::new(
623 trace_id,
624 span_id,
625 TraceFlags::SAMPLED,
626 true,
627 Default::default(),
628 );
629 let expected_trace_id = span_context.trace_id();
630 parent.otel_context = Context::current().with_remote_span_context(span_context);
631
632 let fragments = split_body_lines()(&parent).unwrap();
634 assert!(!fragments.is_empty(), "Should have at least one fragment");
635
636 for fragment in &fragments {
638 let span = fragment.otel_context.span();
639 let frag_span_ctx = span.span_context();
640 assert!(
641 frag_span_ctx.is_valid(),
642 "Fragment should have valid span context"
643 );
644 assert_eq!(
645 frag_span_ctx.trace_id(),
646 expected_trace_id,
647 "Fragment should have same trace ID as parent"
648 );
649 }
650 }
651
652 #[test]
653 fn test_stream_split_config_defaults_valid() {
654 let config = StreamSplitConfig::default();
655 assert!(config.validate().is_ok());
656 }
657
658 #[test]
659 fn test_stream_split_config_batch_size_zero_rejected() {
660 let config = StreamSplitConfig {
661 batch_size: 0,
662 ..Default::default()
663 };
664 let err = config.validate().unwrap_err();
665 assert!(err.to_string().contains("batch_size"));
666 }
667
668 #[test]
669 fn test_stream_split_config_max_record_bytes_zero_rejected() {
670 let config = StreamSplitConfig {
671 max_record_bytes: 0,
672 ..Default::default()
673 };
674 let err = config.validate().unwrap_err();
675 assert!(err.to_string().contains("max_record_bytes"));
676 }
677
678 #[test]
679 fn test_stream_split_config_chunks_requires_chunk_size() {
680 let config = StreamSplitConfig {
681 format: StreamSplitFormat::Chunks,
682 chunk_size: None,
683 ..Default::default()
684 };
685 let err = config.validate().unwrap_err();
686 assert!(err.to_string().contains("Chunks requires chunk_size"));
687 }
688
689 #[test]
690 fn test_stream_split_config_chunk_size_zero_rejected() {
691 let config = StreamSplitConfig {
692 format: StreamSplitFormat::Chunks,
693 chunk_size: Some(0),
694 ..Default::default()
695 };
696 let err = config.validate().unwrap_err();
697 assert!(err.to_string().contains("chunk_size must be > 0"));
698 }
699
700 #[test]
701 fn test_stream_split_config_chunk_size_exceeds_max_record_bytes() {
702 let config = StreamSplitConfig {
703 format: StreamSplitFormat::Chunks,
704 chunk_size: Some(2000),
705 max_record_bytes: 1000,
706 ..Default::default()
707 };
708 let err = config.validate().unwrap_err();
709 assert!(
710 err.to_string()
711 .contains("chunk_size must be <= max_record_bytes")
712 );
713 }
714
715 #[test]
716 fn test_stream_split_config_zip_rejects_chunk_size() {
717 let config = StreamSplitConfig {
718 format: StreamSplitFormat::Zip,
719 chunk_size: Some(1024),
720 ..Default::default()
721 };
722 let err = config.validate().unwrap_err();
723 assert!(err.to_string().contains("Zip does not support chunk_size"));
724 }
725
726 #[test]
727 fn test_all_fragments_share_same_trace_context() {
728 use opentelemetry::Context;
729 use opentelemetry::trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId};
730
731 let mut parent = Exchange::new(Message::new("line1\nline2\nline3"));
733 let trace_id =
734 TraceId::from_bytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x3B, 0x9A, 0xCA, 0x09]);
735 let span_id = SpanId::from_bytes([0, 0, 0, 0, 0, 0, 0, 111]);
736 let span_context = SpanContext::new(
737 trace_id,
738 span_id,
739 TraceFlags::SAMPLED,
740 true,
741 Default::default(),
742 );
743 parent.otel_context = Context::current().with_remote_span_context(span_context);
744
745 let fragments = split_body_lines()(&parent).unwrap();
746 assert_eq!(fragments.len(), 3);
747
748 let trace_ids: Vec<_> = fragments
750 .iter()
751 .map(|f| {
752 let span = f.otel_context.span();
753 span.span_context().trace_id()
754 })
755 .collect();
756
757 assert!(
758 trace_ids.iter().all(|&id| id == trace_id),
759 "All fragments should have the same trace ID"
760 );
761 }
762}