Skip to main content

alien_core/resources/
kubernetes_compute.rs

1//! Compute pools on Kubernetes use administrator-managed node capacity.
2//!
3//! Hardware sizes and machine counts are advisory. Execution requirements must
4//! either be translated into Pod placement or rejected before workloads start.
5
6use crate::{
7    instance_catalog::Architecture, CapacityGroup, ComputeCluster, Container, Daemon, Stack,
8    WorkloadPriority, ENV_ALIEN_KUBERNETES_PRIORITY_CLASSES,
9};
10use std::collections::{BTreeMap, HashSet};
11
12/// Validate logical pools without querying or provisioning Kubernetes nodes.
13pub fn validate_kubernetes_compute(stack: &Stack) -> Vec<String> {
14    let mut errors = Vec::new();
15    if let Err(message) = kubernetes_compute_architecture(stack) {
16        errors.push(message);
17    }
18    for (_, entry) in stack.resources() {
19        let Some(cluster) = entry.config.downcast_ref::<ComputeCluster>() else {
20            continue;
21        };
22        let mut names = HashSet::new();
23        if cluster.capacity_groups.is_empty() {
24            errors.push(format!(
25                "ComputeCluster '{}' must declare at least one pool on Kubernetes",
26                cluster.id
27            ));
28        }
29        for pool in &cluster.capacity_groups {
30            if pool.group_id.is_empty() || !names.insert(pool.group_id.as_str()) {
31                errors.push(format!(
32                    "ComputeCluster '{}' has an empty or duplicate pool '{}'",
33                    cluster.id, pool.group_id
34                ));
35            }
36            if pool.min_size > pool.max_size {
37                errors.push(format!(
38                    "ComputeCluster '{}' pool '{}' has min greater than max",
39                    cluster.id, pool.group_id
40                ));
41            }
42            if pool.instance_type.is_some()
43                || pool.nested_virtualization == Some(true)
44                || pool
45                    .profile
46                    .as_ref()
47                    .is_some_and(|profile| profile.gpu.is_some())
48            {
49                errors.push(format!("ComputeCluster '{}' pool '{}' requests instance type, GPU, or nested virtualization; Kubernetes cannot enforce these node requirements", cluster.id, pool.group_id));
50            }
51        }
52        if cluster
53            .dynamic_container_pool
54            .as_ref()
55            .is_some_and(|name| !names.contains(name.as_str()))
56        {
57            errors.push(format!(
58                "ComputeCluster '{}' dynamic container pool does not exist",
59                cluster.id
60            ));
61        }
62        if !cluster.selected_failure_domains.is_empty()
63            || cluster
64                .container_cidr
65                .as_deref()
66                .is_some_and(|cidr| cidr != "10.244.0.0/16")
67        {
68            errors.push(format!("ComputeCluster '{}' configures provider failure domains or a container CIDR; existing Kubernetes networking and node topology are administrator-managed", cluster.id));
69        }
70    }
71    for (_, entry) in stack.resources() {
72        let placement = if let Some(container) = entry.config.downcast_ref::<Container>() {
73            kubernetes_container_pool(stack, container)
74        } else if let Some(daemon) = entry.config.downcast_ref::<Daemon>() {
75            kubernetes_daemon_pool(stack, daemon)
76        } else {
77            continue;
78        };
79        if let Err(message) = placement {
80            errors.push(message);
81        }
82    }
83    errors
84}
85
86/// Architecture shared by source images built for this Kubernetes stack.
87/// Unspecified pools and Workers inherit this build constraint, so their Pods
88/// cannot land on incompatible nodes in a cluster with mixed architectures.
89pub fn kubernetes_compute_architecture(stack: &Stack) -> Result<Option<Architecture>, String> {
90    let mut selected = None;
91    for architecture in stack
92        .resources()
93        .filter_map(|(_, entry)| entry.config.downcast_ref::<ComputeCluster>())
94        .flat_map(|cluster| &cluster.capacity_groups)
95        .filter_map(|pool| pool.profile.as_ref()?.architecture)
96    {
97        if selected.is_some_and(|selected| selected != architecture) {
98            return Err("Kubernetes compute pools require mixed CPU architectures; one platform image cannot satisfy both".to_string());
99        }
100        selected = Some(architecture);
101    }
102    Ok(selected)
103}
104
105/// Match Pod placement to the architecture used by the stack's source images.
106/// This does not associate Workers with a compute pool or reserve node capacity.
107pub fn kubernetes_compute_node_selector(
108    stack: &Stack,
109    pool: Option<&CapacityGroup>,
110) -> Result<Option<BTreeMap<String, String>>, String> {
111    let built_architecture = kubernetes_compute_architecture(stack)?;
112    let architecture = pool
113        .and_then(|pool| pool.profile.as_ref())
114        .and_then(|profile| profile.architecture)
115        .or(built_architecture);
116    Ok(architecture.map(|architecture| {
117        BTreeMap::from([(
118            "kubernetes.io/arch".to_string(),
119            match architecture {
120                Architecture::Arm64 => "arm64",
121                Architecture::X86_64 => "amd64",
122            }
123            .to_string(),
124        )])
125    }))
126}
127
128/// Map a workload's priority level to the PriorityClass the cluster administrator chose for it.
129///
130/// Returns `None` when the workload declares no level or the administrator mapped none, so
131/// pod specs stay exactly as they were. The map comes from the Operator's
132/// `ALIEN_KUBERNETES_PRIORITY_CLASSES` environment variable, which the Helm chart sets only
133/// when `runtime.workloadPriority.classes` has an entry.
134pub fn kubernetes_priority_class_name(
135    priority: Option<WorkloadPriority>,
136) -> Result<Option<String>, String> {
137    priority_class_name_from(
138        priority,
139        std::env::var(ENV_ALIEN_KUBERNETES_PRIORITY_CLASSES)
140            .ok()
141            .as_deref(),
142    )
143}
144
145fn parse_priority_class_map(raw: &str) -> Result<BTreeMap<String, String>, String> {
146    serde_json::from_str(raw).map_err(|error| {
147        format!(
148            "{ENV_ALIEN_KUBERNETES_PRIORITY_CLASSES} must be a JSON object mapping \
149             critical, high, normal and low to PriorityClass names (chart value \
150             runtime.workloadPriority.classes): {error}"
151        )
152    })
153}
154
155fn priority_class_name_from(
156    priority: Option<WorkloadPriority>,
157    raw_map: Option<&str>,
158) -> Result<Option<String>, String> {
159    let (Some(priority), Some(raw)) = (priority, raw_map) else {
160        return Ok(None);
161    };
162    Ok(parse_priority_class_map(raw)?
163        .remove(priority.as_str())
164        .filter(|name| !name.is_empty()))
165}
166
167/// Add the chart value to fix when Kubernetes rejects a pod because its PriorityClass is missing.
168/// Other messages pass through unchanged.
169pub fn kubernetes_priority_class_failure_hint(message: &str) -> String {
170    priority_class_failure_hint_from(
171        message,
172        std::env::var(ENV_ALIEN_KUBERNETES_PRIORITY_CLASSES)
173            .ok()
174            .as_deref(),
175    )
176}
177
178fn priority_class_failure_hint_from(message: &str, raw_map: Option<&str>) -> String {
179    const MARKER: &str = "no PriorityClass with name ";
180    let Some(start) = message.find(MARKER).map(|index| index + MARKER.len()) else {
181        return message.to_string();
182    };
183    let class = message[start..]
184        .split(" was found")
185        .next()
186        .unwrap_or_default()
187        .trim_matches('"');
188    let levels: Vec<String> = raw_map
189        .and_then(|raw| parse_priority_class_map(raw).ok())
190        .map(|map| {
191            map.into_iter()
192                .filter(|(_, name)| name == class)
193                .map(|(level, _)| format!("runtime.workloadPriority.classes.{level}"))
194                .collect()
195        })
196        .unwrap_or_default();
197    let keys = if levels.is_empty() {
198        "runtime.workloadPriority.classes".to_string()
199    } else {
200        levels.join(", ")
201    };
202    format!(
203        "{message}. Create the PriorityClass '{class}' or set {keys} in the Helm values to an \
204         existing PriorityClass (or set runtime.workloadPriority.create=true)"
205    )
206}
207
208/// Resolve placement, inferring a single declared cluster and its general or
209/// sole pool. Legacy stacks without a declaration keep default scheduling.
210pub fn kubernetes_container_pool<'a>(
211    stack: &'a Stack,
212    container: &Container,
213) -> Result<Option<&'a CapacityGroup>, String> {
214    kubernetes_workload_pool(
215        stack,
216        &container.id,
217        container.cluster.as_deref(),
218        container.pool.as_deref(),
219    )
220}
221
222/// Resolve a daemon's pool without changing its per-node DaemonSet behavior.
223pub fn kubernetes_daemon_pool<'a>(
224    stack: &'a Stack,
225    daemon: &Daemon,
226) -> Result<Option<&'a CapacityGroup>, String> {
227    kubernetes_workload_pool(
228        stack,
229        &daemon.id,
230        daemon.cluster.as_deref(),
231        daemon.pool.as_deref(),
232    )
233}
234
235fn kubernetes_workload_pool<'a>(
236    stack: &'a Stack,
237    resource_id: &str,
238    cluster: Option<&str>,
239    pool: Option<&str>,
240) -> Result<Option<&'a CapacityGroup>, String> {
241    let cluster = match cluster {
242        Some(cluster_id) => stack
243            .resources
244            .get(cluster_id)
245            .and_then(|entry| entry.config.downcast_ref::<ComputeCluster>())
246            .ok_or_else(|| {
247                format!("Workload '{resource_id}' references missing ComputeCluster '{cluster_id}'")
248            })?,
249        None => {
250            let mut clusters = stack
251                .resources()
252                .filter_map(|(_, entry)| entry.config.downcast_ref::<ComputeCluster>());
253            let Some(cluster) = clusters.next() else {
254                return if pool.is_some() {
255                    Err(format!(
256                        "Workload '{resource_id}' selects a pool without a ComputeCluster"
257                    ))
258                } else {
259                    Ok(None)
260                };
261            };
262            if clusters.next().is_some() {
263                return Err(format!(
264                    "Workload '{resource_id}' must select a ComputeCluster explicitly"
265                ));
266            }
267            cluster
268        }
269    };
270    let selected = match pool {
271        Some(name) => cluster
272            .capacity_groups
273            .iter()
274            .find(|pool| pool.group_id == name),
275        None => cluster
276            .capacity_groups
277            .iter()
278            .find(|pool| pool.group_id == "general")
279            .or_else(|| (cluster.capacity_groups.len() == 1).then(|| &cluster.capacity_groups[0])),
280    };
281    selected.map(Some).ok_or_else(|| format!("Workload '{resource_id}' references missing or ambiguous pool '{}' in ComputeCluster '{}'", pool.unwrap_or("general"), cluster.id))
282}
283
284/// Resolve the one pool admitted for release-independent containers.
285pub fn kubernetes_dynamic_pool(stack: &Stack) -> Result<Option<&CapacityGroup>, String> {
286    let mut clusters = stack
287        .resources()
288        .filter_map(|(_, entry)| entry.config.downcast_ref::<ComputeCluster>());
289    let Some(cluster) = clusters.next() else {
290        return Ok(None);
291    };
292    if clusters.next().is_some() {
293        return Err("Dynamic containers require exactly one ComputeCluster".to_string());
294    }
295    let name = cluster
296        .dynamic_container_pool
297        .as_deref()
298        .unwrap_or("general");
299    cluster
300        .capacity_groups
301        .iter()
302        .find(|pool| pool.group_id == name)
303        .map(Some)
304        .ok_or_else(|| {
305            format!(
306                "ComputeCluster '{}' has no dynamic container pool '{name}'",
307                cluster.id
308            )
309        })
310}
311
312#[cfg(test)]
313mod tests {
314    use super::*;
315    use crate::{instance_catalog::Architecture, ContainerCode, ResourceLifecycle};
316
317    fn cluster() -> ComputeCluster {
318        ComputeCluster::new("compute".to_string())
319            .capacity_group(CapacityGroup {
320                group_id: "apps".to_string(),
321                instance_type: None,
322                profile: Some(crate::MachineProfile {
323                    cpu: "2".to_string(),
324                    memory_bytes: 4 << 30,
325                    ephemeral_storage_bytes: 20 << 30,
326                    architecture: Some(Architecture::X86_64),
327                    gpu: None,
328                    data_disk: None,
329                }),
330                min_size: 1,
331                max_size: 3,
332                scale_policy: None,
333                nested_virtualization: None,
334                data_disk_gib_range: None,
335            })
336            .dynamic_container_pool("apps".to_string())
337            .build()
338    }
339
340    fn stack(cluster: ComputeCluster) -> Stack {
341        let container = Container::new("api".to_string())
342            .code(ContainerCode::Image {
343                image: "example.test/api:1".to_string(),
344            })
345            .cpu(crate::ResourceSpec {
346                min: "1".to_string(),
347                desired: "1".to_string(),
348            })
349            .memory(crate::ResourceSpec {
350                min: "128Mi".to_string(),
351                desired: "128Mi".to_string(),
352            })
353            .permissions("default".to_string())
354            .cluster("compute".to_string())
355            .pool("apps".to_string())
356            .build();
357        Stack::new("test".to_string())
358            .add(cluster, ResourceLifecycle::Frozen)
359            .add(container, ResourceLifecycle::Live)
360            .build()
361    }
362
363    #[test]
364    fn portable_pool_preserves_placement_and_dynamic_admission() {
365        let stack = stack(cluster());
366        assert!(validate_kubernetes_compute(&stack).is_empty());
367        let pool = kubernetes_dynamic_pool(&stack).unwrap().unwrap();
368        assert_eq!(pool.group_id, "apps");
369        assert_eq!(
370            pool.profile.as_ref().unwrap().architecture,
371            Some(Architecture::X86_64)
372        );
373    }
374
375    #[test]
376    fn rejects_execution_requirements_and_invalid_references() {
377        let mut cluster = cluster();
378        cluster.capacity_groups[0].nested_virtualization = Some(true);
379        cluster.dynamic_container_pool = Some("missing".to_string());
380        let errors = validate_kubernetes_compute(&stack(cluster));
381        assert!(errors
382            .iter()
383            .any(|message| message.contains("nested virtualization")));
384        assert!(errors
385            .iter()
386            .any(|message| message.contains("dynamic container pool")));
387        let mut cluster = self::cluster();
388        cluster.capacity_groups[0].group_id = "other".to_string();
389        assert!(validate_kubernetes_compute(&stack(cluster))
390            .iter()
391            .any(|message| message.contains("pool 'apps'")));
392    }
393
394    #[test]
395    fn implicit_placement_uses_the_declared_pool_architecture() {
396        let stack = stack(cluster());
397        let mut container = stack
398            .resources
399            .get("api")
400            .unwrap()
401            .config
402            .downcast_ref::<Container>()
403            .unwrap()
404            .clone();
405        container.cluster = None;
406        container.pool = None;
407        let pool = kubernetes_container_pool(&stack, &container)
408            .unwrap()
409            .unwrap();
410        assert_eq!(pool.group_id, "apps");
411        assert_eq!(
412            pool.profile.as_ref().unwrap().architecture,
413            Some(Architecture::X86_64)
414        );
415    }
416
417    #[test]
418    fn unspecified_pool_and_unpooled_worker_use_the_build_architecture() {
419        let mut compute = cluster();
420        let mut unspecified = compute.capacity_groups[0].clone();
421        unspecified.group_id = "other".to_string();
422        unspecified.profile.as_mut().unwrap().architecture = None;
423        compute.capacity_groups.push(unspecified.clone());
424        let stack = stack(compute);
425        let expected = Some(BTreeMap::from([(
426            "kubernetes.io/arch".to_string(),
427            "amd64".to_string(),
428        )]));
429        assert_eq!(
430            kubernetes_compute_node_selector(&stack, Some(&unspecified)).unwrap(),
431            expected
432        );
433        assert_eq!(
434            kubernetes_compute_node_selector(&stack, None).unwrap(),
435            expected
436        );
437    }
438
439    #[test]
440    fn priority_class_maps_declared_level_only_when_mapped() {
441        let map = r#"{"critical":"vendor-core","high":""}"#;
442        let class = |level, map| priority_class_name_from(level, map).unwrap();
443        assert_eq!(
444            class(Some(WorkloadPriority::Critical), Some(map)).as_deref(),
445            Some("vendor-core")
446        );
447        assert_eq!(class(Some(WorkloadPriority::High), Some(map)), None);
448        assert_eq!(class(Some(WorkloadPriority::Low), Some(map)), None);
449        assert_eq!(class(None, Some(map)), None);
450        assert_eq!(class(Some(WorkloadPriority::Critical), None), None);
451        assert!(priority_class_name_from(Some(WorkloadPriority::Low), Some("nope")).is_err());
452    }
453
454    #[test]
455    fn missing_priority_class_error_names_class_and_values_key() {
456        let message = r#"FailedCreate: pods "api-1" is forbidden: no PriorityClass with name vendor-core was found"#;
457        let hint = priority_class_failure_hint_from(message, Some(r#"{"critical":"vendor-core"}"#));
458        assert!(hint.contains("'vendor-core'"), "{hint}");
459        assert!(
460            hint.contains("runtime.workloadPriority.classes.critical"),
461            "{hint}"
462        );
463        assert_eq!(
464            priority_class_failure_hint_from("quota exceeded", None),
465            "quota exceeded"
466        );
467    }
468
469    #[test]
470    fn legacy_stack_keeps_default_placement() {
471        let stack = Stack::new("legacy".to_string()).build();
472        assert!(validate_kubernetes_compute(&stack).is_empty());
473        assert!(kubernetes_dynamic_pool(&stack).unwrap().is_none());
474    }
475}