Skip to main content

alien_core/resources/
kubernetes_compute.rs

1//! Compute pools on Kubernetes use administrator-managed node capacity.
2//!
3//! Hardware sizes and machine counts are advisory. Execution requirements must
4//! either be translated into Pod placement or rejected before workloads start.
5
6use crate::{
7    instance_catalog::Architecture, CapacityGroup, ComputeCluster, Container, Daemon, Stack,
8    WorkloadPriority, ENV_ALIEN_KUBERNETES_PRIORITY_CLASSES,
9};
10use std::collections::{BTreeMap, HashSet};
11
12/// Validate logical pools without querying or provisioning Kubernetes nodes.
13pub fn validate_kubernetes_compute(stack: &Stack) -> Vec<String> {
14    let mut errors = Vec::new();
15    if let Err(message) = kubernetes_compute_architecture(stack) {
16        errors.push(message);
17    }
18    for (_, entry) in stack.resources() {
19        let Some(cluster) = entry.config.downcast_ref::<ComputeCluster>() else {
20            continue;
21        };
22        let mut names = HashSet::new();
23        if cluster.capacity_groups.is_empty() {
24            errors.push(format!(
25                "ComputeCluster '{}' must declare at least one pool on Kubernetes",
26                cluster.id
27            ));
28        }
29        for pool in &cluster.capacity_groups {
30            if pool.group_id.is_empty() || !names.insert(pool.group_id.as_str()) {
31                errors.push(format!(
32                    "ComputeCluster '{}' has an empty or duplicate pool '{}'",
33                    cluster.id, pool.group_id
34                ));
35            }
36            if pool.min_size > pool.max_size {
37                errors.push(format!(
38                    "ComputeCluster '{}' pool '{}' has min greater than max",
39                    cluster.id, pool.group_id
40                ));
41            }
42            if pool.instance_type.is_some()
43                || pool.nested_virtualization == Some(true)
44                || pool
45                    .profile
46                    .as_ref()
47                    .is_some_and(|profile| profile.gpu.is_some())
48            {
49                errors.push(format!("ComputeCluster '{}' pool '{}' requests instance type, GPU, or nested virtualization; Kubernetes cannot enforce these node requirements", cluster.id, pool.group_id));
50            }
51        }
52        if cluster
53            .dynamic_container_pool
54            .as_ref()
55            .is_some_and(|name| !names.contains(name.as_str()))
56        {
57            errors.push(format!(
58                "ComputeCluster '{}' dynamic container pool does not exist",
59                cluster.id
60            ));
61        }
62        if !cluster.selected_failure_domains.is_empty()
63            || cluster
64                .container_cidr
65                .as_deref()
66                .is_some_and(|cidr| cidr != "10.244.0.0/16")
67        {
68            errors.push(format!("ComputeCluster '{}' configures provider failure domains or a container CIDR; existing Kubernetes networking and node topology are administrator-managed", cluster.id));
69        }
70    }
71    for (_, entry) in stack.resources() {
72        let placement = if let Some(container) = entry.config.downcast_ref::<Container>() {
73            kubernetes_container_pool(stack, container)
74        } else if let Some(daemon) = entry.config.downcast_ref::<Daemon>() {
75            kubernetes_daemon_pool(stack, daemon)
76        } else {
77            continue;
78        };
79        if let Err(message) = placement {
80            errors.push(message);
81        }
82    }
83    errors
84}
85
86/// Architecture shared by source images built for this Kubernetes stack.
87/// Unspecified pools and Workers inherit this build constraint, so their Pods
88/// cannot land on incompatible nodes in a cluster with mixed architectures.
89pub fn kubernetes_compute_architecture(stack: &Stack) -> Result<Option<Architecture>, String> {
90    let mut selected = None;
91    for architecture in stack
92        .resources()
93        .filter_map(|(_, entry)| entry.config.downcast_ref::<ComputeCluster>())
94        .flat_map(|cluster| &cluster.capacity_groups)
95        .filter_map(|pool| pool.profile.as_ref()?.architecture)
96    {
97        if selected.is_some_and(|selected| selected != architecture) {
98            return Err("Kubernetes compute pools require mixed CPU architectures; one platform image cannot satisfy both".to_string());
99        }
100        selected = Some(architecture);
101    }
102    Ok(selected)
103}
104
105/// The architecture a release records for a workload when it records exactly one; several
106/// (a multi-architecture image) or none mean no constraint.
107pub fn single_built_architecture(built_architectures: &[Architecture]) -> Option<Architecture> {
108    match built_architectures {
109        [only] => Some(*only),
110        _ => None,
111    }
112}
113
114/// Match Pod placement to the architecture of the workload's image.
115/// The pool's profile wins; otherwise a single built architecture recorded by
116/// `alien build` for this workload (`Stack::built_architectures_of`) selects nodes; several built architectures
117/// (a multi-architecture image) or none leave the Pod unconstrained, as releases
118/// built before the build recorded it were.
119/// This does not associate Workers with a compute pool or reserve node capacity.
120pub fn kubernetes_compute_node_selector(
121    stack: &Stack,
122    pool: Option<&CapacityGroup>,
123    built_architectures: &[Architecture],
124) -> Result<Option<BTreeMap<String, String>>, String> {
125    let stack_architecture = kubernetes_compute_architecture(stack)?;
126    let built = single_built_architecture(built_architectures);
127    let architecture = pool
128        .and_then(|pool| pool.profile.as_ref())
129        .and_then(|profile| profile.architecture)
130        .or(stack_architecture)
131        .or(built);
132    Ok(architecture.map(|architecture| {
133        BTreeMap::from([(
134            "kubernetes.io/arch".to_string(),
135            match architecture {
136                Architecture::Arm64 => "arm64",
137                Architecture::X86_64 => "amd64",
138            }
139            .to_string(),
140        )])
141    }))
142}
143
144/// Map a workload's priority level to the PriorityClass the cluster administrator chose for it.
145///
146/// Returns `None` when the workload declares no level or the administrator mapped none, so
147/// pod specs stay exactly as they were. The map comes from the Operator's
148/// `ALIEN_KUBERNETES_PRIORITY_CLASSES` environment variable, which the Helm chart sets only
149/// when `runtime.workloadPriority.classes` has an entry.
150pub fn kubernetes_priority_class_name(
151    priority: Option<WorkloadPriority>,
152) -> Result<Option<String>, String> {
153    priority_class_name_from(
154        priority,
155        std::env::var(ENV_ALIEN_KUBERNETES_PRIORITY_CLASSES)
156            .ok()
157            .as_deref(),
158    )
159}
160
161fn parse_priority_class_map(raw: &str) -> Result<BTreeMap<String, String>, String> {
162    serde_json::from_str(raw).map_err(|error| {
163        format!(
164            "{ENV_ALIEN_KUBERNETES_PRIORITY_CLASSES} must be a JSON object mapping \
165             critical, high, normal and low to PriorityClass names (chart value \
166             runtime.workloadPriority.classes): {error}"
167        )
168    })
169}
170
171fn priority_class_name_from(
172    priority: Option<WorkloadPriority>,
173    raw_map: Option<&str>,
174) -> Result<Option<String>, String> {
175    let (Some(priority), Some(raw)) = (priority, raw_map) else {
176        return Ok(None);
177    };
178    Ok(parse_priority_class_map(raw)?
179        .remove(priority.as_str())
180        .filter(|name| !name.is_empty()))
181}
182
183/// Add the chart value to fix when Kubernetes rejects a pod because its PriorityClass is missing.
184/// Other messages pass through unchanged.
185pub fn kubernetes_priority_class_failure_hint(message: &str) -> String {
186    priority_class_failure_hint_from(
187        message,
188        std::env::var(ENV_ALIEN_KUBERNETES_PRIORITY_CLASSES)
189            .ok()
190            .as_deref(),
191    )
192}
193
194fn priority_class_failure_hint_from(message: &str, raw_map: Option<&str>) -> String {
195    const MARKER: &str = "no PriorityClass with name ";
196    let Some(start) = message.find(MARKER).map(|index| index + MARKER.len()) else {
197        return message.to_string();
198    };
199    let class = message[start..]
200        .split(" was found")
201        .next()
202        .unwrap_or_default()
203        .trim_matches('"');
204    let levels: Vec<String> = raw_map
205        .and_then(|raw| parse_priority_class_map(raw).ok())
206        .map(|map| {
207            map.into_iter()
208                .filter(|(_, name)| name == class)
209                .map(|(level, _)| format!("runtime.workloadPriority.classes.{level}"))
210                .collect()
211        })
212        .unwrap_or_default();
213    let keys = if levels.is_empty() {
214        "runtime.workloadPriority.classes".to_string()
215    } else {
216        levels.join(", ")
217    };
218    format!(
219        "{message}. Create the PriorityClass '{class}' or set {keys} in the Helm values to an \
220         existing PriorityClass (or set runtime.workloadPriority.create=true)"
221    )
222}
223
224/// Resolve placement, inferring a single declared cluster and its general or
225/// sole pool. Legacy stacks without a declaration keep default scheduling.
226pub fn kubernetes_container_pool<'a>(
227    stack: &'a Stack,
228    container: &Container,
229) -> Result<Option<&'a CapacityGroup>, String> {
230    kubernetes_workload_pool(
231        stack,
232        &container.id,
233        container.cluster.as_deref(),
234        container.pool.as_deref(),
235    )
236}
237
238/// Resolve a daemon's pool without changing its per-node DaemonSet behavior.
239pub fn kubernetes_daemon_pool<'a>(
240    stack: &'a Stack,
241    daemon: &Daemon,
242) -> Result<Option<&'a CapacityGroup>, String> {
243    kubernetes_workload_pool(
244        stack,
245        &daemon.id,
246        daemon.cluster.as_deref(),
247        daemon.pool.as_deref(),
248    )
249}
250
251fn kubernetes_workload_pool<'a>(
252    stack: &'a Stack,
253    resource_id: &str,
254    cluster: Option<&str>,
255    pool: Option<&str>,
256) -> Result<Option<&'a CapacityGroup>, String> {
257    let cluster = match cluster {
258        Some(cluster_id) => stack
259            .resources
260            .get(cluster_id)
261            .and_then(|entry| entry.config.downcast_ref::<ComputeCluster>())
262            .ok_or_else(|| {
263                format!("Workload '{resource_id}' references missing ComputeCluster '{cluster_id}'")
264            })?,
265        None => {
266            let mut clusters = stack
267                .resources()
268                .filter_map(|(_, entry)| entry.config.downcast_ref::<ComputeCluster>());
269            let Some(cluster) = clusters.next() else {
270                return if pool.is_some() {
271                    Err(format!(
272                        "Workload '{resource_id}' selects a pool without a ComputeCluster"
273                    ))
274                } else {
275                    Ok(None)
276                };
277            };
278            if clusters.next().is_some() {
279                return Err(format!(
280                    "Workload '{resource_id}' must select a ComputeCluster explicitly"
281                ));
282            }
283            cluster
284        }
285    };
286    let selected = match pool {
287        Some(name) => cluster
288            .capacity_groups
289            .iter()
290            .find(|pool| pool.group_id == name),
291        None => cluster
292            .capacity_groups
293            .iter()
294            .find(|pool| pool.group_id == "general")
295            .or_else(|| (cluster.capacity_groups.len() == 1).then(|| &cluster.capacity_groups[0])),
296    };
297    selected.map(Some).ok_or_else(|| format!("Workload '{resource_id}' references missing or ambiguous pool '{}' in ComputeCluster '{}'", pool.unwrap_or("general"), cluster.id))
298}
299
300/// Resolve the one pool admitted for release-independent containers.
301pub fn kubernetes_dynamic_pool(stack: &Stack) -> Result<Option<&CapacityGroup>, String> {
302    let mut clusters = stack
303        .resources()
304        .filter_map(|(_, entry)| entry.config.downcast_ref::<ComputeCluster>());
305    let Some(cluster) = clusters.next() else {
306        return Ok(None);
307    };
308    if clusters.next().is_some() {
309        return Err("Dynamic containers require exactly one ComputeCluster".to_string());
310    }
311    let name = cluster
312        .dynamic_container_pool
313        .as_deref()
314        .unwrap_or("general");
315    cluster
316        .capacity_groups
317        .iter()
318        .find(|pool| pool.group_id == name)
319        .map(Some)
320        .ok_or_else(|| {
321            format!(
322                "ComputeCluster '{}' has no dynamic container pool '{name}'",
323                cluster.id
324            )
325        })
326}
327
328#[cfg(test)]
329mod tests {
330    use super::*;
331    use crate::{instance_catalog::Architecture, ContainerCode, ResourceLifecycle};
332
333    fn cluster() -> ComputeCluster {
334        ComputeCluster::new("compute".to_string())
335            .capacity_group(CapacityGroup {
336                group_id: "apps".to_string(),
337                instance_type: None,
338                profile: Some(crate::MachineProfile {
339                    cpu: "2".to_string(),
340                    memory_bytes: 4 << 30,
341                    ephemeral_storage_bytes: 20 << 30,
342                    architecture: Some(Architecture::X86_64),
343                    gpu: None,
344                    data_disk: None,
345                }),
346                min_size: 1,
347                max_size: 3,
348                scale_policy: None,
349                nested_virtualization: None,
350                data_disk_gib_range: None,
351            })
352            .dynamic_container_pool("apps".to_string())
353            .build()
354    }
355
356    fn stack(cluster: ComputeCluster) -> Stack {
357        let container = Container::new("api".to_string())
358            .code(ContainerCode::Image {
359                image: "example.test/api:1".to_string(),
360            })
361            .cpu(crate::ResourceSpec {
362                min: "1".to_string(),
363                desired: "1".to_string(),
364            })
365            .memory(crate::ResourceSpec {
366                min: "128Mi".to_string(),
367                desired: "128Mi".to_string(),
368            })
369            .permissions("default".to_string())
370            .cluster("compute".to_string())
371            .pool("apps".to_string())
372            .build();
373        Stack::new("test".to_string())
374            .add(cluster, ResourceLifecycle::Frozen)
375            .add(container, ResourceLifecycle::Live)
376            .build()
377    }
378
379    #[test]
380    fn portable_pool_preserves_placement_and_dynamic_admission() {
381        let stack = stack(cluster());
382        assert!(validate_kubernetes_compute(&stack).is_empty());
383        let pool = kubernetes_dynamic_pool(&stack).unwrap().unwrap();
384        assert_eq!(pool.group_id, "apps");
385        assert_eq!(
386            pool.profile.as_ref().unwrap().architecture,
387            Some(Architecture::X86_64)
388        );
389    }
390
391    #[test]
392    fn rejects_execution_requirements_and_invalid_references() {
393        let mut cluster = cluster();
394        cluster.capacity_groups[0].nested_virtualization = Some(true);
395        cluster.dynamic_container_pool = Some("missing".to_string());
396        let errors = validate_kubernetes_compute(&stack(cluster));
397        assert!(errors
398            .iter()
399            .any(|message| message.contains("nested virtualization")));
400        assert!(errors
401            .iter()
402            .any(|message| message.contains("dynamic container pool")));
403        let mut cluster = self::cluster();
404        cluster.capacity_groups[0].group_id = "other".to_string();
405        assert!(validate_kubernetes_compute(&stack(cluster))
406            .iter()
407            .any(|message| message.contains("pool 'apps'")));
408    }
409
410    #[test]
411    fn implicit_placement_uses_the_declared_pool_architecture() {
412        let stack = stack(cluster());
413        let mut container = stack
414            .resources
415            .get("api")
416            .unwrap()
417            .config
418            .downcast_ref::<Container>()
419            .unwrap()
420            .clone();
421        container.cluster = None;
422        container.pool = None;
423        let pool = kubernetes_container_pool(&stack, &container)
424            .unwrap()
425            .unwrap();
426        assert_eq!(pool.group_id, "apps");
427        assert_eq!(
428            pool.profile.as_ref().unwrap().architecture,
429            Some(Architecture::X86_64)
430        );
431    }
432
433    #[test]
434    fn unspecified_pool_and_unpooled_worker_use_the_build_architecture() {
435        let mut compute = cluster();
436        let mut unspecified = compute.capacity_groups[0].clone();
437        unspecified.group_id = "other".to_string();
438        unspecified.profile.as_mut().unwrap().architecture = None;
439        compute.capacity_groups.push(unspecified.clone());
440        let stack = stack(compute);
441        let expected = Some(BTreeMap::from([(
442            "kubernetes.io/arch".to_string(),
443            "amd64".to_string(),
444        )]));
445        assert_eq!(
446            kubernetes_compute_node_selector(&stack, Some(&unspecified), &[]).unwrap(),
447            expected
448        );
449        assert_eq!(
450            kubernetes_compute_node_selector(&stack, None, &[]).unwrap(),
451            expected
452        );
453    }
454
455    #[test]
456    fn priority_class_maps_declared_level_only_when_mapped() {
457        let map = r#"{"critical":"vendor-core","high":""}"#;
458        let class = |level, map| priority_class_name_from(level, map).unwrap();
459        assert_eq!(
460            class(Some(WorkloadPriority::Critical), Some(map)).as_deref(),
461            Some("vendor-core")
462        );
463        assert_eq!(class(Some(WorkloadPriority::High), Some(map)), None);
464        assert_eq!(class(Some(WorkloadPriority::Low), Some(map)), None);
465        assert_eq!(class(None, Some(map)), None);
466        assert_eq!(class(Some(WorkloadPriority::Critical), None), None);
467        assert!(priority_class_name_from(Some(WorkloadPriority::Low), Some("nope")).is_err());
468    }
469
470    #[test]
471    fn missing_priority_class_error_names_class_and_values_key() {
472        let message = r#"FailedCreate: pods "api-1" is forbidden: no PriorityClass with name vendor-core was found"#;
473        let hint = priority_class_failure_hint_from(message, Some(r#"{"critical":"vendor-core"}"#));
474        assert!(hint.contains("'vendor-core'"), "{hint}");
475        assert!(
476            hint.contains("runtime.workloadPriority.classes.critical"),
477            "{hint}"
478        );
479        assert_eq!(
480            priority_class_failure_hint_from("quota exceeded", None),
481            "quota exceeded"
482        );
483    }
484
485    #[test]
486    fn legacy_stack_keeps_default_placement() {
487        let stack = Stack::new("legacy".to_string()).build();
488        assert!(validate_kubernetes_compute(&stack).is_empty());
489        assert!(kubernetes_dynamic_pool(&stack).unwrap().is_none());
490    }
491
492    fn recorded(mut stack: Stack, architectures: &[Architecture]) -> Stack {
493        if !architectures.is_empty() {
494            stack
495                .built_architectures
496                .insert("web".to_string(), architectures.to_vec());
497        }
498        stack
499    }
500
501    fn empty_stack() -> Stack {
502        Stack::new("test".to_string()).build()
503    }
504
505    fn arch_label(selector: Option<BTreeMap<String, String>>) -> Option<String> {
506        selector.and_then(|labels| labels.get("kubernetes.io/arch").cloned())
507    }
508
509    #[test]
510    fn a_single_built_architecture_selects_matching_nodes() {
511        let stack = recorded(empty_stack(), &[Architecture::X86_64]);
512        let selector =
513            kubernetes_compute_node_selector(&stack, None, stack.built_architectures_of("web"))
514                .unwrap();
515        assert_eq!(arch_label(selector).as_deref(), Some("amd64"));
516        let stack = recorded(empty_stack(), &[Architecture::Arm64]);
517        let selector =
518            kubernetes_compute_node_selector(&stack, None, stack.built_architectures_of("web"))
519                .unwrap();
520        assert_eq!(arch_label(selector).as_deref(), Some("arm64"));
521    }
522
523    #[test]
524    fn a_multi_architecture_build_or_an_older_release_adds_no_constraint() {
525        let both = recorded(empty_stack(), &[Architecture::X86_64, Architecture::Arm64]);
526        assert_eq!(
527            kubernetes_compute_node_selector(&both, None, both.built_architectures_of("web"))
528                .unwrap(),
529            None
530        );
531        // A release built before the build recorded architectures.
532        assert_eq!(
533            kubernetes_compute_node_selector(&empty_stack(), None, &[]).unwrap(),
534            None
535        );
536        // Another workload's record says nothing about this one.
537        let other = recorded(empty_stack(), &[Architecture::Arm64]);
538        assert_eq!(
539            kubernetes_compute_node_selector(&other, None, other.built_architectures_of("worker"))
540                .unwrap(),
541            None
542        );
543    }
544
545    #[test]
546    fn a_pool_profile_still_wins_over_the_built_architecture() {
547        let stack = recorded(stack(cluster()), &[Architecture::Arm64]);
548        let pool = kubernetes_dynamic_pool(&stack).unwrap();
549        let selector =
550            kubernetes_compute_node_selector(&stack, pool, stack.built_architectures_of("web"))
551                .unwrap();
552        assert_eq!(arch_label(selector).as_deref(), Some("amd64"));
553    }
554}