1use crate::{error::ErrorData, Queue, ResourceRef, Result, Stack, Storage, StorageNotification};
4use alien_error::{AlienError, Context, IntoAlienError};
5use sha2::{Digest, Sha256};
6use std::collections::HashMap;
7
8pub const GCS_SUFFIXES_ATTRIBUTE: &str = "storageEventSuffixes";
11
12pub const GCS_RULE_ATTRIBUTE: &str = "storageNotificationRule";
14
15const GCS_MAX_SUFFIX_ATTRIBUTES: usize = 4;
17
18const GCS_ATTRIBUTE_VALUE_BYTES: usize = 1024;
21
22fn short_hash(parts: &[&str]) -> String {
23 let mut digest = Sha256::new();
24 for part in parts {
25 digest.update(part.as_bytes());
26 digest.update([0]);
27 }
28 format!("{:x}", digest.finalize())[..16].to_string()
29}
30
31pub fn notification_rule_key(rule: &StorageNotification) -> String {
34 short_hash(&[
35 "rule",
36 &rule.queue.id,
37 rule.prefix.as_deref().unwrap_or_default(),
38 rule.suffix.as_deref().unwrap_or_default(),
39 ])
40}
41
42pub fn azure_subscription_prefix(container: &str) -> String {
45 let head: String = container.chars().take(28).collect();
46 let hash = format!("{:x}", Sha256::digest(container.as_bytes()));
47 format!("{head}-sn-{}-", &hash[..10])
48}
49
50pub fn azure_subscription_name(container: &str, rule: &StorageNotification) -> String {
54 format!(
55 "{}{}",
56 azure_subscription_prefix(container),
57 notification_rule_key(rule)
58 )
59}
60
61#[derive(Debug, Clone, PartialEq, Eq)]
65pub struct GcsNotificationConfig<'a> {
66 pub key: String,
69 pub queue: &'a ResourceRef,
71 pub prefix: Option<&'a str>,
73 pub suffix_attributes: Vec<(String, String)>,
76}
77
78pub fn gcs_notification_configs(storage: &Storage) -> Result<Vec<GcsNotificationConfig<'_>>> {
80 let mut groups: Vec<(&ResourceRef, Option<&str>, Vec<&str>, bool)> = Vec::new();
81 for rule in &storage.notifications {
82 let prefix = rule.prefix.as_deref();
83 let index = match groups
84 .iter()
85 .position(|(queue, other, _, _)| queue.id == rule.queue.id && *other == prefix)
86 {
87 Some(index) => index,
88 None => {
89 groups.push((&rule.queue, prefix, Vec::new(), false));
90 groups.len() - 1
91 }
92 };
93 match rule.suffix.as_deref() {
94 Some(suffix) => groups[index].2.push(suffix),
95 None => groups[index].3 = true,
96 }
97 }
98 groups
99 .into_iter()
100 .map(|(queue, prefix, mut suffixes, unfiltered)| {
101 suffixes.sort_unstable();
103 let suffix_attributes = if unfiltered {
104 Vec::new()
105 } else {
106 gcs_suffix_attributes(&suffixes).context(ErrorData::GenericError {
107 message: format!(
108 "Storage '{}' notification suffixes for queue '{}' do not fit in GCS attributes",
109 storage.id, queue.id
110 ),
111 })?
112 };
113 Ok(GcsNotificationConfig {
114 key: short_hash(&["gcs", &queue.id, prefix.unwrap_or_default()]),
115 queue,
116 prefix,
117 suffix_attributes,
118 })
119 })
120 .collect()
121}
122
123fn gcs_suffix_attributes(suffixes: &[&str]) -> Result<Vec<(String, String)>> {
124 let encoded =
125 serde_json::to_string(suffixes)
126 .into_alien_error()
127 .context(ErrorData::GenericError {
128 message: "Encode notification suffixes".to_string(),
129 })?;
130 let mut chunks = Vec::new();
131 let mut rest = encoded.as_str();
132 while !rest.is_empty() {
133 let mut end = rest.len().min(GCS_ATTRIBUTE_VALUE_BYTES);
134 while !rest.is_char_boundary(end) {
135 end -= 1;
136 }
137 chunks.push(&rest[..end]);
138 rest = &rest[end..];
139 }
140 if chunks.len() > GCS_MAX_SUFFIX_ATTRIBUTES {
141 return Err(AlienError::new(ErrorData::GenericError {
142 message: format!(
143 "{} encoded bytes need {} attributes; GCS allows {GCS_MAX_SUFFIX_ATTRIBUTES}",
144 encoded.len(),
145 chunks.len()
146 ),
147 }));
148 }
149 Ok(chunks
150 .into_iter()
151 .enumerate()
152 .map(|(index, chunk)| {
153 (
154 format!("{GCS_SUFFIXES_ATTRIBUTE}{index}"),
155 chunk.to_string(),
156 )
157 })
158 .collect())
159}
160
161pub fn gcs_suffixes(attributes: &HashMap<String, String>) -> Result<Option<Vec<String>>> {
164 let mut encoded = String::new();
165 let mut index = 0;
166 while let Some(chunk) = attributes.get(&format!("{GCS_SUFFIXES_ATTRIBUTE}{index}")) {
167 encoded.push_str(chunk);
168 index += 1;
169 }
170 if index == 0 {
171 return Ok(None);
172 }
173 serde_json::from_str(&encoded)
174 .map(Some)
175 .into_alien_error()
176 .context(ErrorData::GenericError {
177 message: "Storage notification suffix attributes are not a JSON string array"
178 .to_string(),
179 })
180}
181
182pub fn queue_receives_storage_notifications(stack: &Stack, queue_id: &str) -> bool {
184 stack.resources().any(|(_, entry)| {
185 entry
186 .config
187 .downcast_ref::<Storage>()
188 .is_some_and(|storage| {
189 storage.notifications.iter().any(|notification| {
190 notification.queue.resource_type == Queue::RESOURCE_TYPE
191 && notification.queue.id == queue_id
192 })
193 })
194 })
195}
196
197pub fn storages_notifying_queue(stack: &Stack, queue_id: &str) -> Vec<String> {
199 let mut storages: Vec<String> = stack
200 .resources()
201 .filter(|(_, entry)| {
202 entry
203 .config
204 .downcast_ref::<Storage>()
205 .is_some_and(|storage| {
206 storage.notifications.iter().any(|notification| {
207 notification.queue.resource_type == Queue::RESOURCE_TYPE
208 && notification.queue.id == queue_id
209 })
210 })
211 })
212 .map(|(id, _)| id.clone())
213 .collect();
214 storages.sort();
215 storages
216}
217
218pub fn notifications_overlap(a: &StorageNotification, b: &StorageNotification) -> bool {
223 let prefixes = match (a.prefix.as_deref(), b.prefix.as_deref()) {
224 (Some(a), Some(b)) => a.starts_with(b) || b.starts_with(a),
225 _ => true,
226 };
227 let suffixes = match (a.suffix.as_deref(), b.suffix.as_deref()) {
228 (Some(a), Some(b)) => a.ends_with(b) || b.ends_with(a),
229 _ => true,
230 };
231 prefixes && suffixes
232}
233
234pub fn aws_notification_id(prefix: &str, storage_id: &str, index: usize) -> String {
237 format!("{}{index}", aws_notification_id_prefix(prefix, storage_id))
238}
239
240pub fn aws_trigger_events(events: &[String]) -> Vec<String> {
242 events
243 .iter()
244 .map(|event| match event.as_str() {
245 "created" => "s3:ObjectCreated:*".to_string(),
246 "deleted" => "s3:ObjectRemoved:*".to_string(),
247 other => format!("s3:{other}"),
248 })
249 .collect()
250}
251
252pub fn aws_trigger_notification_id(function_name: &str, storage_id: &str) -> String {
255 format!("{function_name}-s3-{storage_id}")
256}
257
258pub fn aws_notification_id_prefix(prefix: &str, storage_id: &str) -> String {
260 let hash = format!("{:x}", Sha256::digest(storage_id.as_bytes()));
261 format!("{prefix}-sn-{}-", &hash[..16])
262}
263
264pub const AWS_QUEUE_INGRESS_SID: &str = "StorageNotificationIngress";
266
267#[cfg(test)]
268mod tests {
269 use super::*;
270 use crate::{Queue, ResourceRef};
271 use std::collections::HashSet;
272
273 fn notification(prefix: Option<&str>, suffix: Option<&str>) -> StorageNotification {
274 StorageNotification::builder()
275 .queue(ResourceRef::new(Queue::RESOURCE_TYPE, "work"))
276 .maybe_prefix(prefix.map(str::to_string))
277 .maybe_suffix(suffix.map(str::to_string))
278 .build()
279 }
280
281 fn filters() -> Vec<Option<String>> {
282 let mut all = vec![None];
283 let mut level = vec![String::new()];
284 for _ in 0..3 {
285 let next: Vec<_> = level
286 .iter()
287 .flat_map(|value| ['a', 'b', 'é'].map(|character| format!("{value}{character}")))
288 .collect();
289 all.extend(next.iter().cloned().map(Some));
290 level = next;
291 }
292 all
293 }
294
295 fn keys() -> Vec<String> {
296 let mut all = vec![String::new()];
297 let mut level = vec![String::new()];
298 for _ in 0..6 {
299 let next: Vec<_> = level
300 .iter()
301 .flat_map(|key| ['a', 'b', 'é'].map(|character| format!("{key}{character}")))
302 .collect();
303 all.extend(next.clone());
304 level = next;
305 }
306 all
307 }
308
309 fn matches(rule: &StorageNotification, key: &str) -> bool {
310 rule.matches_key(key)
311 }
312
313 #[test]
317 fn overlap_equals_existence_of_a_key_matching_both_rules() {
318 let keys = keys();
319 let filters = filters();
320 for (ap, asf) in filters
321 .iter()
322 .flat_map(|p| filters.iter().map(move |s| (p, s)))
323 {
324 let a = notification(ap.as_deref(), asf.as_deref());
325 for (bp, bsf) in filters
326 .iter()
327 .flat_map(|p| filters.iter().map(move |s| (p, s)))
328 {
329 let b = notification(bp.as_deref(), bsf.as_deref());
330 let witnessed = keys.iter().any(|key| matches(&a, key) && matches(&b, key));
331 assert_eq!(notifications_overlap(&a, &b), witnessed, "a={a:?} b={b:?}");
332 }
333 }
334 }
335
336 fn storage(rules: Vec<StorageNotification>) -> Storage {
337 Storage::new("uploads".into()).notifications(rules).build()
338 }
339
340 type Decoded<'a> = (&'a str, Option<&'a str>, Option<Vec<String>>);
342
343 fn delivered_to<'a>(configs: &[Decoded<'a>], key: &str) -> Vec<&'a str> {
346 configs
347 .iter()
348 .filter(|(_, prefix, suffixes)| {
349 prefix.is_none_or(|prefix| key.starts_with(prefix))
350 && suffixes
351 .as_ref()
352 .is_none_or(|suffixes| suffixes.iter().any(|suffix| key.ends_with(suffix)))
353 })
354 .map(|(queue, _, _)| *queue)
355 .collect()
356 }
357
358 fn decoded(storage: &Storage) -> Vec<Decoded<'_>> {
359 gcs_notification_configs(storage)
360 .unwrap()
361 .into_iter()
362 .map(|config| {
363 let attributes: HashMap<_, _> = config.suffix_attributes.into_iter().collect();
364 (
365 config.queue.id.as_str(),
366 config.prefix,
367 gcs_suffixes(&attributes).unwrap(),
368 )
369 })
370 .collect()
371 }
372
373 #[test]
376 fn grouped_gcs_configs_deliver_exactly_the_keys_each_rule_matches() {
377 let filters: Vec<_> = filters()
378 .into_iter()
379 .filter(|filter| {
380 filter
381 .as_ref()
382 .is_none_or(|value| value.chars().count() <= 2)
383 })
384 .collect();
385 let keys: Vec<_> = keys()
386 .into_iter()
387 .filter(|key| key.chars().count() <= 4)
388 .collect();
389 let rules: Vec<_> = filters
390 .iter()
391 .flat_map(|prefix| {
392 filters
393 .iter()
394 .map(move |suffix| notification(prefix.as_deref(), suffix.as_deref()))
395 })
396 .collect();
397 for (index, a) in rules.iter().enumerate() {
398 for b in &rules[index + 1..] {
399 if notifications_overlap(a, b) {
400 continue;
401 }
402 let storage = storage(vec![a.clone(), b.clone()]);
403 let configs = decoded(&storage);
404 assert_eq!(configs.len(), if a.prefix == b.prefix { 1 } else { 2 });
405 for key in &keys {
406 let expected = [a, b].iter().filter(|rule| matches(rule, key)).count();
407 assert_eq!(
408 delivered_to(&configs, key).len(),
409 expected,
410 "a={a:?} b={b:?} key={key:?}"
411 );
412 }
413 }
414 }
415 }
416
417 #[test]
418 fn gcs_configs_split_by_queue_and_keep_unfiltered_rules_unfiltered() {
419 let rule = |queue: &str, prefix: &str, suffix: Option<&str>| {
420 StorageNotification::builder()
421 .queue(ResourceRef::new(Queue::RESOURCE_TYPE, queue))
422 .prefix(prefix.into())
423 .maybe_suffix(suffix.map(str::to_string))
424 .build()
425 };
426 let storage = storage(vec![
427 rule("work", "logs/", Some(".json")),
428 rule("audit", "logs/", Some(".csv")),
429 rule("work", "logs/", Some(".gz")),
430 rule("work", "raw/", None),
431 ]);
432 let configs = decoded(&storage);
433 assert_eq!(
434 configs,
435 [
436 (
437 "work",
438 Some("logs/"),
439 Some(vec![".gz".into(), ".json".into()])
440 ),
441 ("audit", Some("logs/"), Some(vec![".csv".into()])),
442 ("work", Some("raw/"), None),
443 ]
444 );
445 assert_eq!(delivered_to(&configs, "logs/a.json"), ["work"]);
446 assert_eq!(delivered_to(&configs, "logs/a.csv"), ["audit"]);
447 assert!(delivered_to(&configs, "logs/a.txt").is_empty());
448 assert_eq!(delivered_to(&configs, "raw/anything"), ["work"]);
449 }
450
451 #[test]
455 fn largest_allowed_suffix_set_fits_gcs_attribute_limits_and_round_trips() {
456 let suffixes: Vec<String> = (0..10)
457 .map(|index| {
458 let mut suffix = index.to_string();
459 for character in ['"', '\\', '*', 'é'].iter().cycle() {
460 if suffix.len() + character.len_utf8() > 128 {
461 break;
462 }
463 suffix.push(*character);
464 }
465 suffix
466 })
467 .collect();
468 assert!(suffixes.iter().all(|suffix| suffix.len() >= 127));
469 let borrowed: Vec<&str> = suffixes.iter().map(String::as_str).collect();
470 let attributes = gcs_suffix_attributes(&borrowed).unwrap();
471 assert!(attributes.len() <= GCS_MAX_SUFFIX_ATTRIBUTES);
472 assert!(
473 attributes.len() > 1,
474 "the worst case must exercise chunking"
475 );
476 for (_, value) in &attributes {
477 assert!(value.len() <= GCS_ATTRIBUTE_VALUE_BYTES);
478 }
479 let attributes: HashMap<_, _> = attributes.into_iter().collect();
480 assert_eq!(gcs_suffixes(&attributes).unwrap(), Some(suffixes));
481 assert_eq!(gcs_suffixes(&HashMap::new()).unwrap(), None);
482 }
483
484 #[test]
485 fn ids_are_distinct_per_storage_and_rule_and_carry_the_storage_prefix() {
486 let mut ids = HashSet::new();
487 for storage in ["source", "second", "long-resource-identity"] {
488 let prefix = aws_notification_id_prefix("stack", storage);
489 for index in 0..10 {
490 let id = aws_notification_id("stack", storage, index);
491 assert!(id.starts_with(&prefix));
492 assert!(ids.insert(id));
493 }
494 }
495 assert!(!aws_notification_id("stack", "source", 1)
496 .starts_with(&aws_notification_id_prefix("stack", "second")));
497 }
498}