1#![forbid(unsafe_code)]
3
4use async_trait::async_trait;
5use aws_lambda_events::event::sqs::{BatchItemFailure, SqsBatchResponse, SqsEvent, SqsMessage};
6use futures::{StreamExt, stream};
7use lambda_runtime::{Error as LambdaError, service_fn};
8use std::{
9 collections::{BTreeMap, BTreeSet, VecDeque},
10 fmt,
11 sync::Arc,
12};
13use thiserror::Error;
14use tokio::sync::Mutex;
15
16#[cfg(feature = "jobs")]
17pub mod jobs;
18
19const MAX_BATCH_SIZE: usize = 10_000;
20const MAX_MESSAGE_BYTES: usize = 1024 * 1024;
21const MESSAGE_GROUP_ID: &str = "MessageGroupId";
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25pub struct WorkerConfig {
26 pub max_batch_size: usize,
27 pub max_message_bytes: usize,
28 pub max_concurrency: usize,
29}
30
31impl Default for WorkerConfig {
32 fn default() -> Self {
33 Self {
34 max_batch_size: 10,
35 max_message_bytes: 256 * 1024,
36 max_concurrency: 1,
37 }
38 }
39}
40
41impl WorkerConfig {
42 pub fn validate(self) -> Result<Self, WorkerError> {
43 if !(1..=MAX_BATCH_SIZE).contains(&self.max_batch_size) {
44 return Err(WorkerError::InvalidBatchLimit(self.max_batch_size));
45 }
46 if !(1..=MAX_MESSAGE_BYTES).contains(&self.max_message_bytes) {
47 return Err(WorkerError::InvalidMessageLimit(self.max_message_bytes));
48 }
49 if !(1..=self.max_batch_size).contains(&self.max_concurrency) {
50 return Err(WorkerError::InvalidConcurrency(self.max_concurrency));
51 }
52 Ok(self)
53 }
54}
55
56#[derive(Clone, PartialEq, Eq)]
58pub struct WorkerMessage {
59 pub message_id: String,
60 pub body: String,
61 pub attributes: BTreeMap<String, String>,
62 pub message_group_id: Option<String>,
63}
64
65impl fmt::Debug for WorkerMessage {
66 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
67 formatter
68 .debug_struct("WorkerMessage")
69 .field("message_id", &self.message_id)
70 .field("body_bytes", &self.body.len())
71 .field(
72 "attribute_names",
73 &self.attributes.keys().collect::<Vec<_>>(),
74 )
75 .field("has_message_group_id", &self.message_group_id.is_some())
76 .finish()
77 }
78}
79
80#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct WorkerFailure {
83 code: String,
84}
85
86impl WorkerFailure {
87 #[must_use]
88 pub fn new(code: impl Into<String>) -> Self {
89 let code = code.into();
90 let valid = !code.is_empty()
91 && code.len() <= 64
92 && code.bytes().all(|byte| {
93 byte.is_ascii_lowercase()
94 || byte.is_ascii_digit()
95 || matches!(byte, b'-' | b'_' | b'.')
96 });
97 Self {
98 code: if valid {
99 code
100 } else {
101 "handler_failed".to_owned()
102 },
103 }
104 }
105
106 #[must_use]
107 pub fn code(&self) -> &str {
108 &self.code
109 }
110}
111
112#[async_trait]
113pub trait MessageHandler: Send + Sync + 'static {
114 async fn handle(&self, message: WorkerMessage) -> Result<(), WorkerFailure>;
115}
116
117#[derive(Default)]
123pub struct FakeMessageHandler {
124 attempts: Mutex<Vec<WorkerMessage>>,
125 failures: Mutex<BTreeMap<String, VecDeque<WorkerFailure>>>,
126}
127
128impl FakeMessageHandler {
129 pub async fn fail_next(&self, message_id: impl Into<String>, failure: WorkerFailure) {
130 self.failures
131 .lock()
132 .await
133 .entry(message_id.into())
134 .or_default()
135 .push_back(failure);
136 }
137
138 pub async fn attempts(&self) -> Vec<WorkerMessage> {
139 self.attempts.lock().await.clone()
140 }
141
142 pub async fn clear(&self) {
143 self.attempts.lock().await.clear();
144 self.failures.lock().await.clear();
145 }
146}
147
148impl fmt::Debug for FakeMessageHandler {
149 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
150 formatter
151 .debug_struct("FakeMessageHandler")
152 .finish_non_exhaustive()
153 }
154}
155
156#[async_trait]
157impl MessageHandler for FakeMessageHandler {
158 async fn handle(&self, message: WorkerMessage) -> Result<(), WorkerFailure> {
159 let message_id = message.message_id.clone();
160 self.attempts.lock().await.push(message);
161 let mut failures = self.failures.lock().await;
162 let failure = failures.get_mut(&message_id).and_then(VecDeque::pop_front);
163 if failures.get(&message_id).is_some_and(VecDeque::is_empty) {
164 failures.remove(&message_id);
165 }
166 drop(failures);
167 failure.map_or(Ok(()), Err)
168 }
169}
170
171pub async fn run_sqs_worker<H>(handler: Arc<H>, config: WorkerConfig) -> Result<(), LambdaError>
175where
176 H: MessageHandler,
177{
178 let config = config
179 .validate()
180 .map_err(|error| -> LambdaError { Box::new(error) })?;
181 lambda_runtime::run(service_fn(
182 move |event: lambda_runtime::LambdaEvent<SqsEvent>| {
183 let handler = Arc::clone(&handler);
184 async move {
185 process_sqs_event(event.payload, handler, config)
186 .await
187 .map_err(|error| -> LambdaError { Box::new(error) })
188 }
189 },
190 ))
191 .await
192}
193
194pub async fn process_sqs_event<H>(
199 event: SqsEvent,
200 handler: Arc<H>,
201 config: WorkerConfig,
202) -> Result<SqsBatchResponse, WorkerError>
203where
204 H: MessageHandler,
205{
206 let config = config.validate()?;
207 if event.records.len() > config.max_batch_size {
208 return Err(WorkerError::BatchTooLarge {
209 actual: event.records.len(),
210 maximum: config.max_batch_size,
211 });
212 }
213 validate_message_ids(&event.records)?;
214 let fifo = detect_fifo(&event.records)?;
215 let failures = if fifo {
216 process_fifo(event.records, handler, config).await
217 } else {
218 process_standard(event.records, handler, config).await
219 };
220 let mut response = SqsBatchResponse::default();
221 response.batch_item_failures = failures
222 .into_iter()
223 .map(|item_identifier| {
224 let mut failure = BatchItemFailure::default();
225 failure.item_identifier = item_identifier;
226 failure
227 })
228 .collect();
229 Ok(response)
230}
231
232fn validate_message_ids(records: &[SqsMessage]) -> Result<(), WorkerError> {
233 let mut seen = BTreeSet::new();
234 for (index, record) in records.iter().enumerate() {
235 let Some(message_id) = record
236 .message_id
237 .as_deref()
238 .map(str::trim)
239 .filter(|message_id| !message_id.is_empty())
240 else {
241 return Err(WorkerError::MissingMessageId { index });
242 };
243 if !seen.insert(message_id) {
244 return Err(WorkerError::DuplicateMessageId {
245 message_id: message_id.to_owned(),
246 });
247 }
248 }
249 Ok(())
250}
251
252fn detect_fifo(records: &[SqsMessage]) -> Result<bool, WorkerError> {
253 let fifo = records.iter().any(|record| {
254 record.attributes.contains_key(MESSAGE_GROUP_ID)
255 || record
256 .event_source_arn
257 .as_deref()
258 .and_then(|arn| arn.rsplit(':').next())
259 .and_then(|queue| queue.rsplit_once('.'))
260 .is_some_and(|(_, extension)| extension.eq_ignore_ascii_case("fifo"))
261 });
262 if fifo
263 && records
264 .iter()
265 .any(|record| !record.attributes.contains_key(MESSAGE_GROUP_ID))
266 {
267 return Err(WorkerError::IncompleteFifoMetadata);
268 }
269 Ok(fifo)
270}
271
272async fn process_fifo<H>(
273 records: Vec<SqsMessage>,
274 handler: Arc<H>,
275 config: WorkerConfig,
276) -> Vec<String>
277where
278 H: MessageHandler,
279{
280 let mut failures = Vec::new();
281 let mut stopped = false;
282 for record in records {
283 let message_id = record.message_id.clone().expect("validated message ID");
284 if stopped {
285 failures.push(message_id);
286 continue;
287 }
288 match worker_message(record, config.max_message_bytes) {
289 Ok(message) => {
290 if let Err(error) = handler.handle(message).await {
291 tracing::warn!(
292 message_id,
293 failure_code = error.code(),
294 "SQS handler rejected FIFO message"
295 );
296 failures.push(message_id);
297 stopped = true;
298 }
299 }
300 Err(reason) => {
301 tracing::warn!(message_id, reason, "invalid FIFO SQS message");
302 failures.push(message_id);
303 stopped = true;
304 }
305 }
306 }
307 failures
308}
309
310async fn process_standard<H>(
311 records: Vec<SqsMessage>,
312 handler: Arc<H>,
313 config: WorkerConfig,
314) -> Vec<String>
315where
316 H: MessageHandler,
317{
318 let results = stream::iter(records.into_iter().enumerate())
319 .map(|(index, record)| {
320 let handler = Arc::clone(&handler);
321 async move {
322 let message_id = record.message_id.clone().expect("validated message ID");
323 let failed = match worker_message(record, config.max_message_bytes) {
324 Ok(message) => match handler.handle(message).await {
325 Ok(()) => false,
326 Err(error) => {
327 tracing::warn!(
328 message_id,
329 failure_code = error.code(),
330 "SQS handler rejected message"
331 );
332 true
333 }
334 },
335 Err(reason) => {
336 tracing::warn!(message_id, reason, "invalid SQS message");
337 true
338 }
339 };
340 (index, message_id, failed)
341 }
342 })
343 .buffer_unordered(config.max_concurrency)
344 .collect::<Vec<_>>()
345 .await;
346 let mut failures = results
347 .into_iter()
348 .filter(|(_, _, failed)| *failed)
349 .map(|(index, message_id, _)| (index, message_id))
350 .collect::<Vec<_>>();
351 failures.sort_by_key(|(index, _)| *index);
352 failures
353 .into_iter()
354 .map(|(_, message_id)| message_id)
355 .collect()
356}
357
358fn worker_message(
359 record: SqsMessage,
360 max_message_bytes: usize,
361) -> Result<WorkerMessage, &'static str> {
362 let message_id = record.message_id.expect("validated message ID");
363 let body = record
364 .body
365 .filter(|body| !body.is_empty())
366 .ok_or("empty_body")?;
367 if body.len() > max_message_bytes {
368 return Err("body_too_large");
369 }
370 let attributes = record.attributes.into_iter().collect::<BTreeMap<_, _>>();
371 let message_group_id = attributes.get(MESSAGE_GROUP_ID).cloned();
372 Ok(WorkerMessage {
373 message_id,
374 body,
375 attributes,
376 message_group_id,
377 })
378}
379
380#[cfg(feature = "events")]
381pub async fn dispatch_outbox_once(
382 events: &minco_plugin_events::EventServices,
383 worker_id: &str,
384 limit: usize,
385 lease: std::time::Duration,
386) -> Result<minco_plugin_events::DispatchReport, minco_plugin_events::EventError> {
387 let lease = chrono::TimeDelta::from_std(lease)
388 .map_err(|_| minco_plugin_events::EventError::InvalidClaim)?;
389 events.dispatch_once(worker_id, limit, lease).await
390}
391
392#[derive(Debug, Error, PartialEq, Eq)]
393pub enum WorkerError {
394 #[error("SQS batch limit must be between 1 and {MAX_BATCH_SIZE}; found {0}")]
395 InvalidBatchLimit(usize),
396 #[error("SQS message limit must be between 1 and {MAX_MESSAGE_BYTES}; found {0}")]
397 InvalidMessageLimit(usize),
398 #[error("worker concurrency must be between 1 and the configured batch limit; found {0}")]
399 InvalidConcurrency(usize),
400 #[error("SQS batch has {actual} records, exceeding configured maximum {maximum}")]
401 BatchTooLarge { actual: usize, maximum: usize },
402 #[error("SQS record {index} has no usable message ID")]
403 MissingMessageId { index: usize },
404 #[error("SQS batch repeats message ID {message_id}")]
405 DuplicateMessageId { message_id: String },
406 #[error("FIFO SQS records must all provide MessageGroupId")]
407 IncompleteFifoMetadata,
408}
409
410#[cfg(test)]
411mod tests {
412 use super::*;
413 use std::sync::atomic::{AtomicUsize, Ordering};
414 use tokio::sync::Mutex;
415
416 #[derive(Debug, Default)]
417 struct RecordingHandler {
418 seen: Mutex<Vec<String>>,
419 failures: BTreeSet<String>,
420 active: AtomicUsize,
421 maximum_active: AtomicUsize,
422 }
423
424 #[async_trait]
425 impl MessageHandler for RecordingHandler {
426 async fn handle(&self, message: WorkerMessage) -> Result<(), WorkerFailure> {
427 let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;
428 self.maximum_active.fetch_max(active, Ordering::SeqCst);
429 tokio::task::yield_now().await;
430 self.seen.lock().await.push(message.message_id.clone());
431 self.active.fetch_sub(1, Ordering::SeqCst);
432 if self.failures.contains(&message.message_id) {
433 Err(WorkerFailure::new("application_rejected"))
434 } else {
435 Ok(())
436 }
437 }
438 }
439
440 fn record(id: Option<&str>, body: Option<&str>) -> SqsMessage {
441 let mut record = SqsMessage::default();
442 record.message_id = id.map(str::to_owned);
443 record.body = body.map(str::to_owned);
444 record
445 }
446
447 fn event(records: Vec<SqsMessage>) -> SqsEvent {
448 let mut event = SqsEvent::default();
449 event.records = records;
450 event
451 }
452
453 #[test]
454 fn message_debug_redacts_payload_and_attribute_values() {
455 let message = WorkerMessage {
456 message_id: "message-1".to_owned(),
457 body: "body-secret-value".to_owned(),
458 attributes: BTreeMap::from([(
459 "Authorization".to_owned(),
460 "attribute-secret-value".to_owned(),
461 )]),
462 message_group_id: Some("group-secret-value".to_owned()),
463 };
464 let rendered = format!("{message:?}");
465 assert!(rendered.contains("message-1"));
466 assert!(rendered.contains("Authorization"));
467 assert!(!rendered.contains("body-secret-value"));
468 assert!(!rendered.contains("attribute-secret-value"));
469 assert!(!rendered.contains("group-secret-value"));
470 }
471
472 #[tokio::test]
473 async fn mixed_success_and_failure_returns_only_failed_ids_in_input_order() {
474 let handler = Arc::new(RecordingHandler {
475 failures: ["second".to_owned(), "fourth".to_owned()]
476 .into_iter()
477 .collect(),
478 ..RecordingHandler::default()
479 });
480 let response = process_sqs_event(
481 event(vec![
482 record(Some("first"), Some("1")),
483 record(Some("second"), Some("2")),
484 record(Some("third"), Some("3")),
485 record(Some("fourth"), Some("4")),
486 ]),
487 handler,
488 WorkerConfig {
489 max_batch_size: 10,
490 max_message_bytes: 16,
491 max_concurrency: 3,
492 },
493 )
494 .await
495 .unwrap();
496 assert_eq!(
497 response
498 .batch_item_failures
499 .into_iter()
500 .map(|failure| failure.item_identifier)
501 .collect::<Vec<_>>(),
502 ["second", "fourth"]
503 );
504 }
505
506 #[tokio::test]
507 async fn missing_ids_fail_the_invocation() {
508 let result = process_sqs_event(
509 event(vec![record(None, Some("body"))]),
510 Arc::new(RecordingHandler::default()),
511 WorkerConfig::default(),
512 )
513 .await;
514 assert_eq!(result, Err(WorkerError::MissingMessageId { index: 0 }));
515 }
516
517 #[tokio::test]
518 async fn empty_and_oversized_bodies_are_partial_failures() {
519 let response = process_sqs_event(
520 event(vec![
521 record(Some("empty"), None),
522 record(Some("large"), Some("12345")),
523 record(Some("valid"), Some("1234")),
524 ]),
525 Arc::new(RecordingHandler::default()),
526 WorkerConfig {
527 max_batch_size: 3,
528 max_message_bytes: 4,
529 max_concurrency: 1,
530 },
531 )
532 .await
533 .unwrap();
534 assert_eq!(
535 response
536 .batch_item_failures
537 .into_iter()
538 .map(|failure| failure.item_identifier)
539 .collect::<Vec<_>>(),
540 ["empty", "large"]
541 );
542 }
543
544 #[tokio::test]
545 async fn configured_concurrency_is_bounded_and_fully_awaited() {
546 let handler = Arc::new(RecordingHandler::default());
547 let response = process_sqs_event(
548 event(
549 (0..8)
550 .map(|index| record(Some(&format!("m{index}")), Some("body")))
551 .collect(),
552 ),
553 Arc::clone(&handler),
554 WorkerConfig {
555 max_batch_size: 8,
556 max_message_bytes: 16,
557 max_concurrency: 2,
558 },
559 )
560 .await
561 .unwrap();
562 assert!(response.batch_item_failures.is_empty());
563 assert!(handler.maximum_active.load(Ordering::SeqCst) <= 2);
564 assert_eq!(handler.seen.lock().await.len(), 8);
565 }
566
567 #[tokio::test]
568 async fn fifo_stops_after_first_failure_and_fails_forward() {
569 let handler = Arc::new(RecordingHandler {
570 failures: std::iter::once("second".to_owned()).collect(),
571 ..RecordingHandler::default()
572 });
573 let mut records = ["first", "second", "third"]
574 .into_iter()
575 .map(|id| record(Some(id), Some("body")))
576 .collect::<Vec<_>>();
577 for record in &mut records {
578 record
579 .attributes
580 .insert(MESSAGE_GROUP_ID.to_owned(), "group-a".to_owned());
581 }
582 let response = process_sqs_event(
583 event(records),
584 Arc::clone(&handler),
585 WorkerConfig::default(),
586 )
587 .await
588 .unwrap();
589 assert_eq!(*handler.seen.lock().await, ["first", "second"]);
590 assert_eq!(
591 response
592 .batch_item_failures
593 .into_iter()
594 .map(|failure| failure.item_identifier)
595 .collect::<Vec<_>>(),
596 ["second", "third"]
597 );
598 }
599}