1use std::pin::Pin;
2use std::sync::Arc;
3
4use futures::Stream;
5
6use crate::body::{Body, body_type_name};
7use crate::error::CamelError;
8use crate::exchange::Exchange;
9use crate::message::Message;
10
11pub type SplitExpression =
17 Arc<dyn Fn(&Exchange) -> Result<Vec<Exchange>, CamelError> + Send + Sync>;
18
19pub type StreamingSplitExpression = Arc<
26 dyn Fn(Exchange) -> Pin<Box<dyn Stream<Item = Result<Exchange, CamelError>> + Send>>
27 + Send
28 + Sync,
29>;
30
31pub fn streaming_split_type_error(body: &Body) -> CamelError {
35 CamelError::TypeConversionFailed(format!(
36 "streaming split requires body type stream, got {}; add an unmarshal step before split",
37 body_type_name(body)
38 ))
39}
40
41#[derive(Clone, Default)]
43#[non_exhaustive]
44pub enum AggregationStrategy {
45 #[default]
47 LastWins,
48 CollectAll,
50 Original,
52 Custom(Arc<dyn Fn(Exchange, Exchange) -> Exchange + Send + Sync>),
54}
55
56impl std::fmt::Debug for AggregationStrategy {
57 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
58 match self {
59 AggregationStrategy::LastWins => f.write_str("LastWins"),
60 AggregationStrategy::CollectAll => f.write_str("CollectAll"),
61 AggregationStrategy::Original => f.write_str("Original"),
62 AggregationStrategy::Custom(_) => f.write_str("Custom(..)"),
63 }
64 }
65}
66
67#[derive(
69 Clone,
70 Debug,
71 Default,
72 PartialEq,
73 Eq,
74 serde::Serialize,
75 serde::Deserialize,
76 schemars::JsonSchema,
77 ts_rs::TS,
78)]
79#[serde(rename_all = "snake_case")]
80#[ts(rename_all = "snake_case")]
81#[non_exhaustive]
82pub enum StreamSplitFormat {
83 #[default]
85 Auto,
86 Ndjson,
88 Lines,
90 Chunks,
92 Zip,
94 Tar,
96 #[serde(rename = "tar.gz")]
98 #[ts(rename = "tar.gz")]
99 TarGz,
100}
101
102#[derive(
107 Clone,
108 Debug,
109 PartialEq,
110 Eq,
111 serde::Serialize,
112 serde::Deserialize,
113 schemars::JsonSchema,
114 ts_rs::TS,
115)]
116#[serde(rename_all = "snake_case")]
117#[ts(rename_all = "snake_case")]
118pub struct StreamSplitConfig {
119 pub format: StreamSplitFormat,
121 pub max_record_bytes: usize,
123 pub batch_size: usize,
125 pub chunk_size: Option<usize>,
127 pub include_origin: bool,
129}
130
131impl Default for StreamSplitConfig {
132 fn default() -> Self {
133 Self {
134 format: StreamSplitFormat::Auto,
135 max_record_bytes: 1024 * 1024,
136 batch_size: 1,
137 chunk_size: None,
138 include_origin: true,
139 }
140 }
141}
142
143impl StreamSplitConfig {
144 pub fn validate(&self) -> Result<(), CamelError> {
157 if self.batch_size == 0 {
158 return Err(CamelError::Config(
159 "stream split batch_size must be > 0".into(),
160 ));
161 }
162 if self.max_record_bytes == 0 {
163 return Err(CamelError::Config(
164 "stream split max_record_bytes must be > 0".into(),
165 ));
166 }
167 if self.format == StreamSplitFormat::Chunks && self.chunk_size.is_none() {
168 return Err(CamelError::Config(
169 "stream split format=Chunks requires chunk_size".into(),
170 ));
171 }
172 if self.format == StreamSplitFormat::Zip && self.chunk_size.is_some() {
177 return Err(CamelError::Config(
178 "stream split format=Zip does not support chunk_size".into(),
179 ));
180 }
181 if matches!(
182 self.format,
183 StreamSplitFormat::Tar | StreamSplitFormat::TarGz
184 ) && self.chunk_size.is_some()
185 {
186 return Err(CamelError::Config(format!(
187 "stream split format={:?} is a materialized archive format and does not support chunk_size",
188 self.format
189 )));
190 }
191 if let Some(cs) = self.chunk_size
192 && cs == 0
193 {
194 return Err(CamelError::Config(
195 "stream split chunk_size must be > 0".into(),
196 ));
197 }
198 if self.format == StreamSplitFormat::Chunks
199 && let Some(cs) = self.chunk_size
200 && cs > self.max_record_bytes
201 {
202 return Err(CamelError::Config(
203 "stream split chunk_size must be <= max_record_bytes".into(),
204 ));
205 }
206 Ok(())
207 }
208}
209
210#[derive(Clone)]
212pub struct SplitterConfig {
213 pub expression: SplitExpression,
215 pub aggregation: AggregationStrategy,
217 pub parallel: bool,
219 pub parallel_limit: Option<usize>,
221 pub stop_on_exception: bool,
227 pub max_fragments: usize,
233}
234
235impl std::fmt::Debug for SplitterConfig {
236 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
237 f.debug_struct("SplitterConfig")
238 .field("expression", &"<split-expression>")
239 .field("aggregation", &self.aggregation)
240 .field("parallel", &self.parallel)
241 .field("parallel_limit", &self.parallel_limit)
242 .field("stop_on_exception", &self.stop_on_exception)
243 .field("max_fragments", &self.max_fragments)
244 .finish()
245 }
246}
247
248impl SplitterConfig {
249 pub fn new(expression: SplitExpression) -> Self {
251 Self {
252 expression,
253 aggregation: AggregationStrategy::default(),
254 parallel: false,
255 parallel_limit: None,
256 stop_on_exception: true,
257 max_fragments: 100_000,
258 }
259 }
260
261 pub fn aggregation(mut self, strategy: AggregationStrategy) -> Self {
263 self.aggregation = strategy;
264 self
265 }
266
267 pub fn parallel(mut self, parallel: bool) -> Self {
269 self.parallel = parallel;
270 self
271 }
272
273 pub fn parallel_limit(mut self, limit: usize) -> Self {
275 self.parallel_limit = Some(limit);
276 self
277 }
278
279 pub fn stop_on_exception(mut self, stop: bool) -> Self {
284 self.stop_on_exception = stop;
285 self
286 }
287
288 pub fn max_fragments(mut self, max: usize) -> Self {
290 self.max_fragments = max;
291 self
292 }
293
294 pub fn validate(&self) -> Result<(), CamelError> {
299 if self.parallel && self.parallel_limit == Some(0) {
300 return Err(CamelError::Config(
301 "splitter parallel_limit must be > 0".to_string(),
302 ));
303 }
304 if self.max_fragments == 0 {
305 return Err(CamelError::Config(
306 "splitter max_fragments must be > 0".to_string(),
307 ));
308 }
309 Ok(())
310 }
311}
312
313pub fn fragment_exchange(parent: &Exchange, body: Body) -> Exchange {
338 let mut msg = Message::new(body);
339 msg.headers = parent.input.headers.clone();
340 let mut ex = Exchange::new(msg);
341 ex.properties = parent.properties.clone();
342 ex.pattern = parent.pattern;
343 ex.otel_context = parent.otel_context.clone();
345 ex
346}
347
348pub fn split_body_lines() -> SplitExpression {
354 Arc::new(|exchange: &Exchange| {
355 let text = match &exchange.input.body {
356 Body::Text(s) => s.as_str(),
357 Body::Empty => return Ok(Vec::new()),
358 _ => {
359 return Err(CamelError::TypeConversionFailed(format!(
360 "split expression 'body_lines' requires body type text, got {received}; add an unmarshal step before split",
361 received = body_type_name(&exchange.input.body)
362 )));
363 }
364 };
365 Ok(text
366 .lines()
367 .map(|line| fragment_exchange(exchange, Body::Text(line.to_string())))
368 .collect())
369 })
370}
371
372pub fn split_body_json_array() -> SplitExpression {
378 Arc::new(|exchange: &Exchange| {
379 let arr = match &exchange.input.body {
380 Body::Json(serde_json::Value::Array(arr)) => arr,
381 Body::Empty => return Ok(Vec::new()),
382 Body::Json(_) => {
383 return Err(CamelError::TypeConversionFailed(
384 "split expression 'body_json_array' requires body type json (array), got json (non-array); add an unmarshal step before split"
385 .to_string(),
386 ))
387 }
388 _ => {
389 return Err(CamelError::TypeConversionFailed(format!(
390 "split expression 'body_json_array' requires body type json (array), got {received}; add an unmarshal step before split",
391 received = body_type_name(&exchange.input.body)
392 )))
393 }
394 };
395 Ok(arr
396 .iter()
397 .map(|val| fragment_exchange(exchange, Body::Json(val.clone())))
398 .collect())
399 })
400}
401
402pub fn split_body<F>(f: F) -> SplitExpression
407where
408 F: Fn(&Body) -> Vec<Body> + Send + Sync + 'static,
409{
410 Arc::new(move |exchange: &Exchange| {
411 Ok(f(&exchange.input.body)
412 .into_iter()
413 .map(|body| fragment_exchange(exchange, body))
414 .collect())
415 })
416}
417
418#[cfg(test)]
419mod tests {
420 use super::*;
421 use crate::value::Value;
422
423 #[test]
424 fn test_split_body_lines() {
425 let mut ex = Exchange::new(Message::new("a\nb\nc"));
426 ex.input.set_header("source", Value::String("test".into()));
427 ex.set_property("trace", Value::Bool(true));
428
429 let fragments = split_body_lines()(&ex).unwrap();
430 assert_eq!(fragments.len(), 3);
431 assert_eq!(fragments[0].input.body.as_text(), Some("a"));
432 assert_eq!(fragments[1].input.body.as_text(), Some("b"));
433 assert_eq!(fragments[2].input.body.as_text(), Some("c"));
434
435 for frag in &fragments {
437 assert_eq!(
438 frag.input.header("source"),
439 Some(&Value::String("test".into()))
440 );
441 assert_eq!(frag.property("trace"), Some(&Value::Bool(true)));
442 }
443 }
444
445 #[test]
446 fn test_split_body_lines_empty() {
447 let ex = Exchange::new(Message::default()); let fragments = split_body_lines()(&ex).unwrap();
449 assert!(fragments.is_empty());
450 }
451
452 #[test]
453 fn test_split_body_json_array() {
454 let arr = serde_json::json!([1, 2, 3]);
455 let ex = Exchange::new(Message::new(arr));
456
457 let fragments = split_body_json_array()(&ex).unwrap();
458 assert_eq!(fragments.len(), 3);
459 assert!(matches!(&fragments[0].input.body, Body::Json(v) if *v == serde_json::json!(1)));
460 assert!(matches!(&fragments[1].input.body, Body::Json(v) if *v == serde_json::json!(2)));
461 assert!(matches!(&fragments[2].input.body, Body::Json(v) if *v == serde_json::json!(3)));
462 }
463
464 #[test]
465 fn test_split_body_json_array_not_array() {
466 let obj = serde_json::json!({"not": "array"});
467 let ex = Exchange::new(Message::new(obj));
468
469 let err = split_body_json_array()(&ex).unwrap_err();
470 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
471 assert!(err.to_string().contains("json (non-array)"));
472 }
473
474 #[test]
475 fn test_split_body_lines_wrong_type_json_errors() {
476 let ex = Exchange::new(Message::new(serde_json::json!({"a": 1})));
477
478 let err = split_body_lines()(&ex).unwrap_err();
479 let msg = err.to_string();
480 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
481 for needle in [
482 "body_lines",
483 "json",
484 "text",
485 "add an unmarshal step before split",
486 ] {
487 assert!(msg.contains(needle), "message '{msg}' missing '{needle}'");
488 }
489 }
490
491 #[test]
492 fn test_split_body_json_array_wrong_type_text_errors() {
493 let ex = Exchange::new(Message::new("x"));
494
495 let err = split_body_json_array()(&ex).unwrap_err();
496 let msg = err.to_string();
497 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
498 for needle in [
499 "body_json_array",
500 "text",
501 "json (array)",
502 "add an unmarshal step before split",
503 ] {
504 assert!(msg.contains(needle), "message '{msg}' missing '{needle}'");
505 }
506 }
507
508 #[test]
509 fn test_split_body_json_array_non_array_json_errors() {
510 let ex = Exchange::new(Message::new(serde_json::json!({"o": 1})));
511
512 let err = split_body_json_array()(&ex).unwrap_err();
513 let msg = err.to_string();
514 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
515 assert!(msg.contains("json (non-array)"));
516 }
517
518 #[test]
519 fn test_split_body_lines_empty_body_ok() {
520 let ex = Exchange::new(Message::default()); let fragments = split_body_lines()(&ex).unwrap();
522 assert!(fragments.is_empty());
523 }
524
525 #[test]
526 fn test_split_body_json_array_empty_body_ok() {
527 let ex = Exchange::new(Message::default()); let fragments = split_body_json_array()(&ex).unwrap();
529 assert!(fragments.is_empty());
530 }
531
532 #[test]
533 fn test_split_body_json_array_empty_array_ok() {
534 let ex = Exchange::new(Message::new(serde_json::json!([])));
535 let fragments = split_body_json_array()(&ex).unwrap();
536 assert!(fragments.is_empty());
537 }
538
539 #[test]
540 fn test_split_body_lines_empty_text_ok() {
541 let ex = Exchange::new(Message::new(""));
542 let fragments = split_body_lines()(&ex).unwrap();
543 assert!(fragments.is_empty());
544 }
545
546 #[test]
547 fn test_split_error_omits_payload() {
548 let ex = Exchange::new(Message::new(serde_json::json!({
549 "secret": "SECRET-8f31a"
550 })));
551
552 let err = split_body_lines()(&ex).unwrap_err();
553 let msg = err.to_string();
554 assert!(matches!(err, CamelError::TypeConversionFailed(_)));
555 for needle in [
556 "body_lines",
557 "json",
558 "text",
559 "add an unmarshal step before split",
560 ] {
561 assert!(msg.contains(needle), "message '{msg}' missing '{needle}'");
562 }
563 assert!(
564 !msg.contains("SECRET-8f31a"),
565 "message '{msg}' leaks payload"
566 );
567 }
568
569 #[test]
570 fn test_split_body_custom() {
571 let splitter = split_body(|body: &Body| match body {
572 Body::Text(s) => s
573 .split(',')
574 .map(|part| Body::Text(part.trim().to_string()))
575 .collect(),
576 _ => Vec::new(),
577 });
578
579 let mut ex = Exchange::new(Message::new("x, y, z"));
580 ex.set_property("id", Value::from(42));
581
582 let fragments = splitter(&ex).unwrap();
583 assert_eq!(fragments.len(), 3);
584 assert_eq!(fragments[0].input.body.as_text(), Some("x"));
585 assert_eq!(fragments[1].input.body.as_text(), Some("y"));
586 assert_eq!(fragments[2].input.body.as_text(), Some("z"));
587
588 for frag in &fragments {
590 assert_eq!(frag.property("id"), Some(&Value::from(42)));
591 }
592 }
593
594 #[test]
595 fn test_splitter_config_defaults() {
596 let config = SplitterConfig::new(split_body_lines());
597 assert!(matches!(config.aggregation, AggregationStrategy::LastWins));
598 assert!(!config.parallel);
599 assert!(config.parallel_limit.is_none());
600 assert!(config.stop_on_exception);
601 }
602
603 #[test]
604 fn test_splitter_config_builder() {
605 let config = SplitterConfig::new(split_body_lines())
606 .aggregation(AggregationStrategy::CollectAll)
607 .parallel(true)
608 .parallel_limit(4)
609 .stop_on_exception(false);
610
611 assert!(matches!(
612 config.aggregation,
613 AggregationStrategy::CollectAll
614 ));
615 assert!(config.parallel);
616 assert_eq!(config.parallel_limit, Some(4));
617 assert!(!config.stop_on_exception);
618 }
619
620 #[test]
621 fn test_splitter_config_default_max_fragments() {
622 let cfg = SplitterConfig::new(Arc::new(|_: &Exchange| Ok(Vec::new())) as SplitExpression);
623 assert_eq!(cfg.max_fragments, 100_000);
624 }
625
626 #[test]
627 fn test_splitter_config_rejects_zero_max_fragments() {
628 let cfg = SplitterConfig::new(Arc::new(|_: &Exchange| Ok(Vec::new())) as SplitExpression)
629 .max_fragments(0);
630 assert!(cfg.validate().is_err());
631 }
632
633 #[test]
634 fn test_fragment_exchange_inherits_otel_context() {
635 use opentelemetry::Context;
636 use opentelemetry::trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId};
637
638 let mut parent = Exchange::new(Message::new("test"));
640 let trace_id = TraceId::from_bytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123]);
641 let span_id = SpanId::from_bytes([0, 0, 0, 0, 0, 0, 1, 200]);
642 let span_context = SpanContext::new(
643 trace_id,
644 span_id,
645 TraceFlags::SAMPLED,
646 true,
647 Default::default(),
648 );
649 let expected_trace_id = span_context.trace_id();
650 parent.otel_context = Context::current().with_remote_span_context(span_context);
651
652 let fragments = split_body_lines()(&parent).unwrap();
654 assert!(!fragments.is_empty(), "Should have at least one fragment");
655
656 for fragment in &fragments {
658 let span = fragment.otel_context.span();
659 let frag_span_ctx = span.span_context();
660 assert!(
661 frag_span_ctx.is_valid(),
662 "Fragment should have valid span context"
663 );
664 assert_eq!(
665 frag_span_ctx.trace_id(),
666 expected_trace_id,
667 "Fragment should have same trace ID as parent"
668 );
669 }
670 }
671
672 #[test]
673 fn test_stream_split_config_defaults_valid() {
674 let config = StreamSplitConfig::default();
675 assert!(config.validate().is_ok());
676 }
677
678 #[test]
679 fn test_stream_split_config_batch_size_zero_rejected() {
680 let config = StreamSplitConfig {
681 batch_size: 0,
682 ..Default::default()
683 };
684 let err = config.validate().unwrap_err();
685 assert!(err.to_string().contains("batch_size"));
686 }
687
688 #[test]
689 fn test_stream_split_config_max_record_bytes_zero_rejected() {
690 let config = StreamSplitConfig {
691 max_record_bytes: 0,
692 ..Default::default()
693 };
694 let err = config.validate().unwrap_err();
695 assert!(err.to_string().contains("max_record_bytes"));
696 }
697
698 #[test]
699 fn test_stream_split_config_chunks_requires_chunk_size() {
700 let config = StreamSplitConfig {
701 format: StreamSplitFormat::Chunks,
702 chunk_size: None,
703 ..Default::default()
704 };
705 let err = config.validate().unwrap_err();
706 assert!(err.to_string().contains("Chunks requires chunk_size"));
707 }
708
709 #[test]
710 fn test_stream_split_config_chunk_size_zero_rejected() {
711 let config = StreamSplitConfig {
712 format: StreamSplitFormat::Chunks,
713 chunk_size: Some(0),
714 ..Default::default()
715 };
716 let err = config.validate().unwrap_err();
717 assert!(err.to_string().contains("chunk_size must be > 0"));
718 }
719
720 #[test]
721 fn test_stream_split_config_chunk_size_exceeds_max_record_bytes() {
722 let config = StreamSplitConfig {
723 format: StreamSplitFormat::Chunks,
724 chunk_size: Some(2000),
725 max_record_bytes: 1000,
726 ..Default::default()
727 };
728 let err = config.validate().unwrap_err();
729 assert!(
730 err.to_string()
731 .contains("chunk_size must be <= max_record_bytes")
732 );
733 }
734
735 #[test]
736 fn test_stream_split_config_zip_rejects_chunk_size() {
737 let config = StreamSplitConfig {
738 format: StreamSplitFormat::Zip,
739 chunk_size: Some(1024),
740 ..Default::default()
741 };
742 let err = config.validate().unwrap_err();
743 assert!(err.to_string().contains("Zip does not support chunk_size"));
744 }
745
746 #[test]
747 fn test_all_fragments_share_same_trace_context() {
748 use opentelemetry::Context;
749 use opentelemetry::trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId};
750
751 let mut parent = Exchange::new(Message::new("line1\nline2\nline3"));
753 let trace_id =
754 TraceId::from_bytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x3B, 0x9A, 0xCA, 0x09]);
755 let span_id = SpanId::from_bytes([0, 0, 0, 0, 0, 0, 0, 111]);
756 let span_context = SpanContext::new(
757 trace_id,
758 span_id,
759 TraceFlags::SAMPLED,
760 true,
761 Default::default(),
762 );
763 parent.otel_context = Context::current().with_remote_span_context(span_context);
764
765 let fragments = split_body_lines()(&parent).unwrap();
766 assert_eq!(fragments.len(), 3);
767
768 let trace_ids: Vec<_> = fragments
770 .iter()
771 .map(|f| {
772 let span = f.otel_context.span();
773 span.span_context().trace_id()
774 })
775 .collect();
776
777 assert!(
778 trace_ids.iter().all(|&id| id == trace_id),
779 "All fragments should have the same trace ID"
780 );
781 }
782}