Skip to main content

alien_core/
storage_notifications.rs

1//! Shared notification translation for setup generators and live controllers.
2
3use crate::{error::ErrorData, Queue, ResourceRef, Result, Stack, Storage, StorageNotification};
4use alien_error::{AlienError, Context, IntoAlienError};
5use sha2::{Digest, Sha256};
6use std::collections::HashMap;
7
8/// Custom GCS notification attributes `storageEventSuffixes0`, `storageEventSuffixes1`, …
9/// whose values, concatenated in order, form a JSON array of the config's literal suffixes.
10pub const GCS_SUFFIXES_ATTRIBUTE: &str = "storageEventSuffixes";
11
12/// Stable config identity for additive GCS notification reconciliation.
13pub const GCS_RULE_ATTRIBUTE: &str = "storageNotificationRule";
14
15/// GCS accepts at most 5 custom attributes per notification config; one is the rule.
16const GCS_MAX_SUFFIX_ATTRIBUTES: usize = 4;
17
18/// GCS rejects custom attribute values longer than 1024 characters, and Pub/Sub attribute
19/// values are limited to 1024 bytes. Chunks are cut at character boundaries by bytes.
20const GCS_ATTRIBUTE_VALUE_BYTES: usize = 1024;
21
22fn short_hash(parts: &[&str]) -> String {
23    let mut digest = Sha256::new();
24    for part in parts {
25        digest.update(part.as_bytes());
26        digest.update([0]);
27    }
28    format!("{:x}", digest.finalize())[..16].to_string()
29}
30
31/// Stable identity of what a rule delivers (queue, prefix, suffix), independent of its
32/// position, so adding or removing another rule never replaces this one's subscription.
33pub fn notification_rule_key(rule: &StorageNotification) -> String {
34    short_hash(&[
35        "rule",
36        &rule.queue.id,
37        rule.prefix.as_deref().unwrap_or_default(),
38        rule.suffix.as_deref().unwrap_or_default(),
39    ])
40}
41
42/// Every Event Grid subscription name of this container's rules starts with this. Azure
43/// container names are ASCII; the hash keeps long names distinct after truncation.
44pub fn azure_subscription_prefix(container: &str) -> String {
45    let head: String = container.chars().take(28).collect();
46    let hash = format!("{:x}", Sha256::digest(container.as_bytes()));
47    format!("{head}-sn-{}-", &hash[..10])
48}
49
50/// The name of the Event Grid subscription that delivers one rule of the storage in
51/// `container`. It carries the hash of what the rule delivers, not its position. At most 59
52/// characters.
53pub fn azure_subscription_name(container: &str, rule: &StorageNotification) -> String {
54    format!(
55        "{}{}",
56        azure_subscription_prefix(container),
57        notification_rule_key(rule)
58    )
59}
60
61/// One GCS notification config: every rule of a Storage with the same queue and prefix.
62/// GCS filters by prefix only, so one config per rule would publish each object under a
63/// shared prefix once per rule. The consumer checks the suffixes instead.
64#[derive(Debug, Clone, PartialEq, Eq)]
65pub struct GcsNotificationConfig<'a> {
66    /// Stable identity of the group (queue and prefix), independent of rule positions and
67    /// suffixes: a changed suffix set replaces only this group's config.
68    pub key: String,
69    /// Queue the config publishes to.
70    pub queue: &'a ResourceRef,
71    /// `object_name_prefix`, if any.
72    pub prefix: Option<&'a str>,
73    /// Suffix attributes; empty when a rule of the group has no suffix and so matches
74    /// every object under the prefix (rules can't overlap, so it is then the only rule).
75    pub suffix_attributes: Vec<(String, String)>,
76}
77
78/// GCS notification configs for a Storage's rules, in order of first appearance.
79pub fn gcs_notification_configs(storage: &Storage) -> Result<Vec<GcsNotificationConfig<'_>>> {
80    let mut groups: Vec<(&ResourceRef, Option<&str>, Vec<&str>, bool)> = Vec::new();
81    for rule in &storage.notifications {
82        let prefix = rule.prefix.as_deref();
83        let index = match groups
84            .iter()
85            .position(|(queue, other, _, _)| queue.id == rule.queue.id && *other == prefix)
86        {
87            Some(index) => index,
88            None => {
89                groups.push((&rule.queue, prefix, Vec::new(), false));
90                groups.len() - 1
91            }
92        };
93        match rule.suffix.as_deref() {
94            Some(suffix) => groups[index].2.push(suffix),
95            None => groups[index].3 = true,
96        }
97    }
98    groups
99        .into_iter()
100        .map(|(queue, prefix, mut suffixes, unfiltered)| {
101            // Sorted, so reordering rules leaves the group's config unchanged.
102            suffixes.sort_unstable();
103            let suffix_attributes = if unfiltered {
104                Vec::new()
105            } else {
106                gcs_suffix_attributes(&suffixes).context(ErrorData::GenericError {
107                    message: format!(
108                        "Storage '{}' notification suffixes for queue '{}' do not fit in GCS attributes",
109                        storage.id, queue.id
110                    ),
111                })?
112            };
113            Ok(GcsNotificationConfig {
114                key: short_hash(&["gcs", &queue.id, prefix.unwrap_or_default()]),
115                queue,
116                prefix,
117                suffix_attributes,
118            })
119        })
120        .collect()
121}
122
123fn gcs_suffix_attributes(suffixes: &[&str]) -> Result<Vec<(String, String)>> {
124    let encoded =
125        serde_json::to_string(suffixes)
126            .into_alien_error()
127            .context(ErrorData::GenericError {
128                message: "Encode notification suffixes".to_string(),
129            })?;
130    let mut chunks = Vec::new();
131    let mut rest = encoded.as_str();
132    while !rest.is_empty() {
133        let mut end = rest.len().min(GCS_ATTRIBUTE_VALUE_BYTES);
134        while !rest.is_char_boundary(end) {
135            end -= 1;
136        }
137        chunks.push(&rest[..end]);
138        rest = &rest[end..];
139    }
140    if chunks.len() > GCS_MAX_SUFFIX_ATTRIBUTES {
141        return Err(AlienError::new(ErrorData::GenericError {
142            message: format!(
143                "{} encoded bytes need {} attributes; GCS allows {GCS_MAX_SUFFIX_ATTRIBUTES}",
144                encoded.len(),
145                chunks.len()
146            ),
147        }));
148    }
149    Ok(chunks
150        .into_iter()
151        .enumerate()
152        .map(|(index, chunk)| {
153            (
154                format!("{GCS_SUFFIXES_ATTRIBUTE}{index}"),
155                chunk.to_string(),
156            )
157        })
158        .collect())
159}
160
161/// The literal suffixes a GCS notification's attributes carry, or `None` when they carry
162/// none (every object under the config's prefix matches).
163pub fn gcs_suffixes(attributes: &HashMap<String, String>) -> Result<Option<Vec<String>>> {
164    let mut encoded = String::new();
165    let mut index = 0;
166    while let Some(chunk) = attributes.get(&format!("{GCS_SUFFIXES_ATTRIBUTE}{index}")) {
167        encoded.push_str(chunk);
168        index += 1;
169    }
170    if index == 0 {
171        return Ok(None);
172    }
173    serde_json::from_str(&encoded)
174        .map(Some)
175        .into_alien_error()
176        .context(ErrorData::GenericError {
177            message: "Storage notification suffix attributes are not a JSON string array"
178                .to_string(),
179        })
180}
181
182/// Whether this queue owns an ingress grant for storage notifications in its stack.
183pub fn queue_receives_storage_notifications(stack: &Stack, queue_id: &str) -> bool {
184    stack.resources().any(|(_, entry)| {
185        entry
186            .config
187            .downcast_ref::<Storage>()
188            .is_some_and(|storage| {
189                storage.notifications.iter().any(|notification| {
190                    notification.queue.resource_type == Queue::RESOURCE_TYPE
191                        && notification.queue.id == queue_id
192                })
193            })
194    })
195}
196
197/// Ids of the Storages in the stack that notify this queue, sorted for deterministic output.
198pub fn storages_notifying_queue(stack: &Stack, queue_id: &str) -> Vec<String> {
199    let mut storages: Vec<String> = stack
200        .resources()
201        .filter(|(_, entry)| {
202            entry
203                .config
204                .downcast_ref::<Storage>()
205                .is_some_and(|storage| {
206                    storage.notifications.iter().any(|notification| {
207                        notification.queue.resource_type == Queue::RESOURCE_TYPE
208                            && notification.queue.id == queue_id
209                    })
210                })
211        })
212        .map(|(id, _)| id.clone())
213        .collect();
214    storages.sort();
215    storages
216}
217
218/// Whether one object key can match both rules. Rules on one Storage must not overlap:
219/// S3 rejects such a configuration ("Configuration is ambiguously defined"), so the
220/// portable declaration rejects it everywhere. Prefixes overlap when either is absent or
221/// one starts with the other; suffixes when either is absent or one ends with the other.
222pub fn notifications_overlap(a: &StorageNotification, b: &StorageNotification) -> bool {
223    let prefixes = match (a.prefix.as_deref(), b.prefix.as_deref()) {
224        (Some(a), Some(b)) => a.starts_with(b) || b.starts_with(a),
225        _ => true,
226    };
227    let suffixes = match (a.suffix.as_deref(), b.suffix.as_deref()) {
228        (Some(a), Some(b)) => a.ends_with(b) || b.ends_with(a),
229        _ => true,
230    };
231    prefixes && suffixes
232}
233
234/// Id of a rule's S3 queue configuration. The stack prefix and a hash of the Storage's
235/// identity let the owning controller recognise its own entries among everyone else's.
236pub fn aws_notification_id(prefix: &str, storage_id: &str, index: usize) -> String {
237    format!("{}{index}", aws_notification_id_prefix(prefix, storage_id))
238}
239
240/// The S3 event names a worker's storage trigger subscribes to.
241pub fn aws_trigger_events(events: &[String]) -> Vec<String> {
242    events
243        .iter()
244        .map(|event| match event.as_str() {
245            "created" => "s3:ObjectCreated:*".to_string(),
246            "deleted" => "s3:ObjectRemoved:*".to_string(),
247            other => format!("s3:{other}"),
248        })
249        .collect()
250}
251
252/// Id of the S3 function configuration (and of the Lambda permission statement) a worker's
253/// storage trigger installs.
254pub fn aws_trigger_notification_id(function_name: &str, storage_id: &str) -> String {
255    format!("{function_name}-s3-{storage_id}")
256}
257
258/// Every [`aws_notification_id`] of one Storage starts with this.
259pub fn aws_notification_id_prefix(prefix: &str, storage_id: &str) -> String {
260    let hash = format!("{:x}", Sha256::digest(storage_id.as_bytes()));
261    format!("{prefix}-sn-{}-", &hash[..16])
262}
263
264/// `Sid` of the queue policy statement that lets S3 deliver a stack's notifications.
265pub const AWS_QUEUE_INGRESS_SID: &str = "StorageNotificationIngress";
266
267#[cfg(test)]
268mod tests {
269    use super::*;
270    use crate::{Queue, ResourceRef};
271    use std::collections::HashSet;
272
273    fn notification(prefix: Option<&str>, suffix: Option<&str>) -> StorageNotification {
274        StorageNotification::builder()
275            .queue(ResourceRef::new(Queue::RESOURCE_TYPE, "work"))
276            .maybe_prefix(prefix.map(str::to_string))
277            .maybe_suffix(suffix.map(str::to_string))
278            .build()
279    }
280
281    fn filters() -> Vec<Option<String>> {
282        let mut all = vec![None];
283        let mut level = vec![String::new()];
284        for _ in 0..3 {
285            let next: Vec<_> = level
286                .iter()
287                .flat_map(|value| ['a', 'b', 'é'].map(|character| format!("{value}{character}")))
288                .collect();
289            all.extend(next.iter().cloned().map(Some));
290            level = next;
291        }
292        all
293    }
294
295    fn keys() -> Vec<String> {
296        let mut all = vec![String::new()];
297        let mut level = vec![String::new()];
298        for _ in 0..6 {
299            let next: Vec<_> = level
300                .iter()
301                .flat_map(|key| ['a', 'b', 'é'].map(|character| format!("{key}{character}")))
302                .collect();
303            all.extend(next.clone());
304            level = next;
305        }
306        all
307    }
308
309    fn matches(rule: &StorageNotification, key: &str) -> bool {
310        rule.matches_key(key)
311    }
312
313    /// Rules overlap exactly when they share a key that S3 would have to route twice.
314    /// Keys are long enough (6 characters vs filters of at most 3) that every
315    /// overlapping pair has a witness key.
316    #[test]
317    fn overlap_equals_existence_of_a_key_matching_both_rules() {
318        let keys = keys();
319        let filters = filters();
320        for (ap, asf) in filters
321            .iter()
322            .flat_map(|p| filters.iter().map(move |s| (p, s)))
323        {
324            let a = notification(ap.as_deref(), asf.as_deref());
325            for (bp, bsf) in filters
326                .iter()
327                .flat_map(|p| filters.iter().map(move |s| (p, s)))
328            {
329                let b = notification(bp.as_deref(), bsf.as_deref());
330                let witnessed = keys.iter().any(|key| matches(&a, key) && matches(&b, key));
331                assert_eq!(notifications_overlap(&a, &b), witnessed, "a={a:?} b={b:?}");
332            }
333        }
334    }
335
336    fn storage(rules: Vec<StorageNotification>) -> Storage {
337        Storage::new("uploads".into()).notifications(rules).build()
338    }
339
340    /// A GCS config as the consumer sees it: queue, prefix and decoded suffixes.
341    type Decoded<'a> = (&'a str, Option<&'a str>, Option<Vec<String>>);
342
343    /// Queues of the configs that deliver `key`: GCS filters by prefix, the consumer by the
344    /// decoded suffixes.
345    fn delivered_to<'a>(configs: &[Decoded<'a>], key: &str) -> Vec<&'a str> {
346        configs
347            .iter()
348            .filter(|(_, prefix, suffixes)| {
349                prefix.is_none_or(|prefix| key.starts_with(prefix))
350                    && suffixes
351                        .as_ref()
352                        .is_none_or(|suffixes| suffixes.iter().any(|suffix| key.ends_with(suffix)))
353            })
354            .map(|(queue, _, _)| *queue)
355            .collect()
356    }
357
358    fn decoded(storage: &Storage) -> Vec<Decoded<'_>> {
359        gcs_notification_configs(storage)
360            .unwrap()
361            .into_iter()
362            .map(|config| {
363                let attributes: HashMap<_, _> = config.suffix_attributes.into_iter().collect();
364                (
365                    config.queue.id.as_str(),
366                    config.prefix,
367                    gcs_suffixes(&attributes).unwrap(),
368                )
369            })
370            .collect()
371    }
372
373    /// For every pair of non-overlapping rules: rules with one queue and prefix share a
374    /// config, and every key is delivered exactly once if a rule matches it, else never.
375    #[test]
376    fn grouped_gcs_configs_deliver_exactly_the_keys_each_rule_matches() {
377        let filters: Vec<_> = filters()
378            .into_iter()
379            .filter(|filter| {
380                filter
381                    .as_ref()
382                    .is_none_or(|value| value.chars().count() <= 2)
383            })
384            .collect();
385        let keys: Vec<_> = keys()
386            .into_iter()
387            .filter(|key| key.chars().count() <= 4)
388            .collect();
389        let rules: Vec<_> = filters
390            .iter()
391            .flat_map(|prefix| {
392                filters
393                    .iter()
394                    .map(move |suffix| notification(prefix.as_deref(), suffix.as_deref()))
395            })
396            .collect();
397        for (index, a) in rules.iter().enumerate() {
398            for b in &rules[index + 1..] {
399                if notifications_overlap(a, b) {
400                    continue;
401                }
402                let storage = storage(vec![a.clone(), b.clone()]);
403                let configs = decoded(&storage);
404                assert_eq!(configs.len(), if a.prefix == b.prefix { 1 } else { 2 });
405                for key in &keys {
406                    let expected = [a, b].iter().filter(|rule| matches(rule, key)).count();
407                    assert_eq!(
408                        delivered_to(&configs, key).len(),
409                        expected,
410                        "a={a:?} b={b:?} key={key:?}"
411                    );
412                }
413            }
414        }
415    }
416
417    #[test]
418    fn gcs_configs_split_by_queue_and_keep_unfiltered_rules_unfiltered() {
419        let rule = |queue: &str, prefix: &str, suffix: Option<&str>| {
420            StorageNotification::builder()
421                .queue(ResourceRef::new(Queue::RESOURCE_TYPE, queue))
422                .prefix(prefix.into())
423                .maybe_suffix(suffix.map(str::to_string))
424                .build()
425        };
426        let storage = storage(vec![
427            rule("work", "logs/", Some(".json")),
428            rule("audit", "logs/", Some(".csv")),
429            rule("work", "logs/", Some(".gz")),
430            rule("work", "raw/", None),
431        ]);
432        let configs = decoded(&storage);
433        assert_eq!(
434            configs,
435            [
436                (
437                    "work",
438                    Some("logs/"),
439                    Some(vec![".gz".into(), ".json".into()])
440                ),
441                ("audit", Some("logs/"), Some(vec![".csv".into()])),
442                ("work", Some("raw/"), None),
443            ]
444        );
445        assert_eq!(delivered_to(&configs, "logs/a.json"), ["work"]);
446        assert_eq!(delivered_to(&configs, "logs/a.csv"), ["audit"]);
447        assert!(delivered_to(&configs, "logs/a.txt").is_empty());
448        assert_eq!(delivered_to(&configs, "raw/anything"), ["work"]);
449    }
450
451    /// The largest suffix set the portable limits allow (10 rules, 128-byte suffixes) must
452    /// fit the GCS limits measured on real GCS: at most 5 custom attributes per config
453    /// (one is the rule) and values of at most 1024 characters.
454    #[test]
455    fn largest_allowed_suffix_set_fits_gcs_attribute_limits_and_round_trips() {
456        let suffixes: Vec<String> = (0..10)
457            .map(|index| {
458                let mut suffix = index.to_string();
459                for character in ['"', '\\', '*', 'é'].iter().cycle() {
460                    if suffix.len() + character.len_utf8() > 128 {
461                        break;
462                    }
463                    suffix.push(*character);
464                }
465                suffix
466            })
467            .collect();
468        assert!(suffixes.iter().all(|suffix| suffix.len() >= 127));
469        let borrowed: Vec<&str> = suffixes.iter().map(String::as_str).collect();
470        let attributes = gcs_suffix_attributes(&borrowed).unwrap();
471        assert!(attributes.len() <= GCS_MAX_SUFFIX_ATTRIBUTES);
472        assert!(
473            attributes.len() > 1,
474            "the worst case must exercise chunking"
475        );
476        for (_, value) in &attributes {
477            assert!(value.len() <= GCS_ATTRIBUTE_VALUE_BYTES);
478        }
479        let attributes: HashMap<_, _> = attributes.into_iter().collect();
480        assert_eq!(gcs_suffixes(&attributes).unwrap(), Some(suffixes));
481        assert_eq!(gcs_suffixes(&HashMap::new()).unwrap(), None);
482    }
483
484    #[test]
485    fn ids_are_distinct_per_storage_and_rule_and_carry_the_storage_prefix() {
486        let mut ids = HashSet::new();
487        for storage in ["source", "second", "long-resource-identity"] {
488            let prefix = aws_notification_id_prefix("stack", storage);
489            for index in 0..10 {
490                let id = aws_notification_id("stack", storage, index);
491                assert!(id.starts_with(&prefix));
492                assert!(ids.insert(id));
493            }
494        }
495        assert!(!aws_notification_id("stack", "source", 1)
496            .starts_with(&aws_notification_id_prefix("stack", "second")));
497    }
498}