Skip to main content

kaniop_operator/controller/
context.rs

1use super::{
2    ControllerId, KanidmClients, idm_reconcile_interval,
3    kanidm::{ClientLockKey, KanidmKey, KanidmResource, KanidmUser},
4};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics::ControllerMetrics;
8use kaniop_k8s_util::error::{Error, Result};
9
10use kaniop_k8s_util::types::short_type_name;
11
12use std::collections::HashMap;
13use std::sync::Arc;
14
15use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder};
16use k8s_openapi::{NamespaceResourceScope, api::core::v1::Namespace};
17use kanidm_client::KanidmClient;
18use kube::runtime::events::{Event, EventType, Recorder};
19use kube::{Api, client::Client};
20use kube::{Resource, ResourceExt};
21use kube::{
22    api::{Patch, PatchParams},
23    runtime::reflector::{Lookup, ObjectRef, Store},
24};
25use serde::{Deserialize, Serialize};
26use tokio::sync::{Mutex, RwLock};
27use tokio::time::Duration;
28use tracing::{debug, error, info, trace};
29
30// Context for our reconciler
31#[derive(Clone)]
32pub struct Context<K: Resource> {
33    /// Controller ID
34    pub controller_id: ControllerId,
35    /// Kubernetes client
36    pub client: Client,
37    /// Prometheus metrics
38    pub metrics: Arc<ControllerMetrics>,
39    /// State of the error backoff policy per object
40    error_backoff_cache: Arc<RwLock<HashMap<ObjectRef<K>, RwLock<ExponentialBackoff>>>>,
41    /// Event recorder
42    pub recorder: Recorder,
43    /// Cache for Namespace resources
44    pub namespace_store: Store<Namespace>,
45    /// Cache for Kanidm resources
46    pub kanidm_store: Store<Kanidm>,
47    /// Shared Kanidm cache clients with the ability to manage users and their groups
48    idm_clients: Arc<RwLock<KanidmClients>>,
49    /// Shared Kanidm cache clients with the ability to manage the operation of Kanidm as a
50    /// database and service
51    system_clients: Arc<RwLock<KanidmClients>>,
52    /// Locks for client creation to prevent thundering herd problem
53    client_creation_locks: Arc<RwLock<HashMap<ClientLockKey, Arc<Mutex<()>>>>>,
54}
55
56impl<K> Context<K>
57where
58    K: Resource + ResourceExt + Lookup + Clone + 'static,
59    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
60{
61    #[allow(clippy::too_many_arguments)]
62    pub fn new(
63        controller_id: ControllerId,
64        client: Client,
65        metrics: Arc<ControllerMetrics>,
66        recorder: Recorder,
67        idm_clients: Arc<RwLock<KanidmClients>>,
68        system_clients: Arc<RwLock<KanidmClients>>,
69        namespace_store: Store<Namespace>,
70        kanidm_store: Store<Kanidm>,
71    ) -> Self {
72        Self {
73            controller_id,
74            client,
75            metrics,
76            recorder,
77            namespace_store,
78            kanidm_store,
79            idm_clients,
80            system_clients,
81            error_backoff_cache: Arc::default(),
82            client_creation_locks: Arc::default(),
83        }
84    }
85
86    pub async fn release_kanidm_clients(&self, kanidm: &Kanidm) {
87        let key = KanidmKey {
88            // safe unwrap: Kanidm is namespaced scoped
89            namespace: kube::ResourceExt::namespace(kanidm).unwrap(),
90            name: kanidm.name_any(),
91        };
92
93        self.idm_clients.write().await.remove(&key);
94        self.system_clients.write().await.remove(&key);
95        {
96            let mut locks = self.client_creation_locks.write().await;
97            locks.remove(&ClientLockKey {
98                namespace: key.namespace.clone(),
99                name: key.name.clone(),
100                user: KanidmUser::IdmAdmin,
101            });
102            locks.remove(&ClientLockKey {
103                namespace: key.namespace.clone(),
104                name: key.name.clone(),
105                user: KanidmUser::Admin,
106            });
107        }
108    }
109
110    /// Return a valid client for the given Kanidm cluster and user, reusing the shared client
111    /// cache. A new client, and therefore a new Kanidm session, is only created when there is no
112    /// cached client or its session is no longer valid.
113    pub async fn get_kanidm_cluster_client(
114        &self,
115        kanidm: &Kanidm,
116        user: KanidmUser,
117    ) -> Result<Arc<KanidmClient>> {
118        // safe unwrap: Kanidm is namespaced scoped
119        let namespace = kube::ResourceExt::namespace(kanidm).unwrap();
120        self.get_cached_client(&namespace, &kanidm.name_any(), user)
121            .await
122    }
123
124    /// Check if a valid client exists in cache
125    async fn get_valid_cached_client(
126        cache: &Arc<RwLock<KanidmClients>>,
127        key: &KanidmKey,
128        namespace: &str,
129        name: &str,
130    ) -> Option<Arc<KanidmClient>> {
131        let client = cache.read().await.get(key).cloned()?;
132
133        trace!(namespace, name, "check existing Kanidm client session");
134        if client.auth_valid().await.is_ok() {
135            trace!(namespace, name, "reuse Kanidm client session");
136            Some(client)
137        } else {
138            None
139        }
140    }
141
142    /// Return a valid client from the cache, creating and caching a new one when needed. This
143    /// operation require to do at least a request for validating the client, use it wisely.
144    async fn get_cached_client(
145        &self,
146        namespace: &str,
147        name: &str,
148        user: KanidmUser,
149    ) -> Result<Arc<KanidmClient>> {
150        debug!(namespace, name, "get Kanidm client");
151
152        let cache = match user {
153            KanidmUser::Admin => self.system_clients.clone(),
154            KanidmUser::IdmAdmin => self.idm_clients.clone(),
155        };
156        let key = KanidmKey {
157            namespace: namespace.to_string(),
158            name: name.to_string(),
159        };
160        if let Some(client) = Self::get_valid_cached_client(&cache, &key, namespace, name).await {
161            return Ok(client);
162        }
163
164        // Slow path: acquire lock for this specific client to prevent concurrent creation
165        let creation_lock = self
166            .client_creation_locks
167            .write()
168            .await
169            .entry(ClientLockKey {
170                namespace: namespace.to_string(),
171                name: name.to_string(),
172                user: user.clone(),
173            })
174            .or_insert_with(Arc::default)
175            .clone();
176        let _guard = creation_lock.lock().await;
177
178        // Double-check: another task may have created the client while we waited for the lock
179        if let Some(client) = Self::get_valid_cached_client(&cache, &key, namespace, name).await {
180            return Ok(client);
181        }
182
183        let client =
184            KanidmClients::create_client(namespace, name, user, self.client.clone()).await?;
185        cache.write().await.insert(key, client.clone());
186        Ok(client)
187    }
188}
189
190impl<K> Context<K>
191where
192    K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
193    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
194{
195    /// Return a valid client for the Kanidm cluster. This operation require to do at least a
196    /// request for validating the client, use it wisely.
197    async fn get_kanidm_client(&self, obj: &K, user: KanidmUser) -> Result<Arc<KanidmClient>> {
198        let namespace = obj.kanidm_namespace();
199        let name = obj.kanidm_name();
200
201        match self.get_cached_client(&namespace, &name, user).await {
202            Ok(client) => Ok(client),
203            Err(e) => {
204                self.recorder
205                    .publish(
206                        &Event {
207                            type_: EventType::Warning,
208                            reason: "KanidmClientError".to_string(),
209                            note: Some(e.to_string()),
210                            action: "KanidmClientCreating".into(),
211                            secondary: None,
212                        },
213                        &obj.object_ref(&()),
214                    )
215                    .await
216                    .map_err(|e| {
217                        error!(%e, "failed to create Kanidm client");
218                        Error::KubeError("failed to publish event".to_string(), Box::new(e))
219                    })?;
220                Err(e)
221            }
222        }
223    }
224
225    /// Return true when normal identity reconciliation may mutate the target Kanidm.
226    pub fn kanidm_write_allowed(&self, obj: &K) -> bool {
227        self.get_kanidm(obj).is_none_or(|kanidm| {
228            !kanidm
229                .annotations()
230                .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
231        })
232    }
233
234    /// Return [`Kanidm`] of the given object
235    ///
236    /// [`Kanidm`]: struct.Kanidm.html
237    pub fn get_kanidm(&self, obj: &K) -> Option<Arc<Kanidm>> {
238        let namespace = obj.kanidm_namespace();
239        let name = obj.kanidm_name();
240        self.kanidm_store.find(|k| {
241            kube::ResourceExt::namespace(k).as_ref() == Some(&namespace) && k.name_any() == name
242        })
243    }
244}
245
246#[allow(async_fn_in_trait)]
247pub trait BackoffContext<K: Resource> {
248    fn metrics(&self) -> &Arc<ControllerMetrics>;
249    async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration;
250    async fn reset_backoff(&self, obj_ref: ObjectRef<K>);
251}
252
253impl<K> BackoffContext<K> for Context<K>
254where
255    K: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
256    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
257{
258    fn metrics(&self) -> &Arc<ControllerMetrics> {
259        &self.metrics
260    }
261
262    /// Return next duration of the backoff policy for the given object
263    async fn get_backoff(&self, obj_ref: ObjectRef<K>) -> Duration {
264        {
265            let read_guard = self.error_backoff_cache.read().await;
266            if let Some(backoff) = read_guard.get(&obj_ref) {
267                if let Some(duration) = backoff.write().await.next() {
268                    return duration;
269                }
270            }
271        }
272
273        // Backoff policy: 1s, 2s, 4s, 8s, 16s, 32s, 64s, 128s, 256s, 300s, 300s...
274        let mut backoff = ExponentialBuilder::default()
275            .with_max_delay(idm_reconcile_interval())
276            .without_max_times()
277            .build();
278        // First backoff is always Some(Duration), but use defensive fallback
279        let duration = backoff.next().unwrap_or_else(|| {
280            trace!("backoff returned None, using default duration");
281            Duration::from_secs(1)
282        });
283        let mut cache = self.error_backoff_cache.write().await;
284        use std::collections::hash_map::Entry;
285        match cache.entry(obj_ref.clone()) {
286            Entry::Vacant(vacant) => {
287                vacant.insert(RwLock::new(backoff));
288                self.metrics.objects_in_backoff_inc();
289            }
290            Entry::Occupied(_occupied) => {}
291        }
292        trace!(
293            namespace = obj_ref.namespace.as_deref().unwrap(),
294            name = obj_ref.name,
295            "recreate backoff policy"
296        );
297        duration
298    }
299
300    /// Reset the backoff policy for the given object
301    async fn reset_backoff(&self, obj_ref: ObjectRef<K>) {
302        let removed = self.error_backoff_cache.write().await.remove(&obj_ref);
303        if removed.is_some() {
304            trace!(
305                namespace = obj_ref.namespace.as_deref().unwrap(),
306                name = obj_ref.name,
307                "reset backoff policy"
308            );
309            self.metrics.objects_in_backoff_dec();
310        }
311    }
312}
313
314#[allow(async_fn_in_trait)]
315pub trait IdmClientContext<K: Resource> {
316    async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
317}
318
319impl<K> IdmClientContext<K> for Context<K>
320where
321    K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
322    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
323{
324    async fn get_idm_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
325        self.get_kanidm_client(obj, KanidmUser::IdmAdmin).await
326    }
327}
328
329#[allow(async_fn_in_trait)]
330pub trait SystemClientContext<K: Resource> {
331    async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>>;
332}
333
334impl<K> SystemClientContext<K> for Context<K>
335where
336    K: Resource<DynamicType = ()> + ResourceExt + KanidmResource + Lookup + Clone + 'static,
337    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
338{
339    async fn get_system_client(&self, obj: &K) -> Result<Arc<KanidmClient>> {
340        self.get_kanidm_client(obj, KanidmUser::Admin).await
341    }
342}
343
344fn is_immutable_update_message(message: &str) -> bool {
345    let message = message.to_ascii_lowercase();
346
347    message.contains("field is immutable")
348        || message.contains("may not change once set")
349        || message.contains("may not be changed once set")
350        || (message.contains("updates to statefulset spec for fields other than")
351            && message.contains("are forbidden"))
352}
353
354#[allow(async_fn_in_trait)]
355pub trait KubeOperations<T, K>
356where
357    T: Resource + ResourceExt + Lookup + Clone + 'static,
358    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
359    K: Resource<Scope = NamespaceResourceScope>
360        + Serialize
361        + Clone
362        + std::fmt::Debug
363        + for<'de> Deserialize<'de>,
364    <K as kube::Resource>::DynamicType: Default,
365    <K as Resource>::Scope: std::marker::Sized,
366{
367    async fn kube_delete(&self, client: Client, metrics: &ControllerMetrics, obj: &K)
368    -> Result<()>;
369    async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K>;
370    async fn kube_patch(
371        &self,
372        client: Client,
373        metrics: &ControllerMetrics,
374        obj: K,
375        operator_name: &str,
376    ) -> Result<K>;
377}
378
379impl<T, K> KubeOperations<T, K> for T
380where
381    T: Resource + ResourceExt + Lookup + Clone + 'static,
382    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
383    K: Resource<Scope = NamespaceResourceScope>
384        + Serialize
385        + Clone
386        + std::fmt::Debug
387        + for<'de> Deserialize<'de>,
388    <K as kube::Resource>::DynamicType: Default,
389    <K as Resource>::Scope: std::marker::Sized,
390{
391    async fn kube_delete(
392        &self,
393        client: Client,
394        _metrics: &ControllerMetrics,
395        obj: &K,
396    ) -> Result<()> {
397        let name = obj.name_any();
398        // safe unwrap: self is namespaced scoped
399        let namespace = kube::ResourceExt::namespace(self).unwrap();
400        trace!(
401            resource.name = &name,
402            resource.namespace = &namespace,
403            "deleting {}",
404            short_type_name::<K>().unwrap_or("Unknown")
405        );
406        let api = Api::<K>::namespaced(client, &namespace);
407        match api.delete(&name, &Default::default()).await {
408            Ok(_) => Ok(()),
409            Err(kube::Error::Api(ae)) if ae.code == 404 => {
410                trace!(
411                    resource.name = &name,
412                    resource.namespace = &namespace,
413                    "{} not found, treating delete as successful",
414                    short_type_name::<K>().unwrap_or("Unknown")
415                );
416                Ok(())
417            }
418            Err(e) => Err(Error::KubeError(
419                format!(
420                    "failed to delete {} {namespace}/{name}",
421                    short_type_name::<K>().unwrap_or("Unknown")
422                ),
423                Box::new(e),
424            )),
425        }
426    }
427
428    async fn kube_apply(&self, client: Client, obj: K, operator_name: &str) -> Result<K> {
429        let name = obj.name_any();
430        // safe unwrap: self is namespaced scoped
431        let namespace = kube::ResourceExt::namespace(self).unwrap();
432        trace!(
433            resource.name = &name,
434            resource.namespace = &namespace,
435            "applying {}",
436            short_type_name::<K>().unwrap_or("Unknown")
437        );
438        let resource_api = Api::<K>::namespaced(client, &namespace);
439
440        resource_api
441            .patch(
442                &name,
443                &PatchParams::apply(operator_name).force(),
444                &Patch::Apply(&obj),
445            )
446            .await
447            .map_err(|e| {
448                Error::KubeError(
449                    format!(
450                        "failed to patch {} {namespace}/{name}",
451                        short_type_name::<K>().unwrap_or("Unknown")
452                    ),
453                    Box::new(e),
454                )
455            })
456    }
457
458    async fn kube_patch(
459        &self,
460        client: Client,
461        metrics: &ControllerMetrics,
462        obj: K,
463        operator_name: &str,
464    ) -> Result<K> {
465        let name = obj.name_any();
466        // safe unwrap: self is namespaced scoped
467        let namespace = kube::ResourceExt::namespace(self).unwrap();
468        let result = self
469            .kube_apply(client.clone(), obj.clone(), operator_name)
470            .await;
471
472        if let Err(Error::KubeError(_, cause)) = &result
473            && let kube::Error::Api(ae) = cause.as_ref()
474            && ae.code == 422
475            && is_immutable_update_message(&ae.message)
476        {
477            info!(resource.name = &name, resource.namespace = &namespace, reason = %ae.reason, api_message = %ae.message, "recreating {} because an immutable field changed",
478                    short_type_name::<K>().unwrap_or("Unknown"));
479            self.kube_delete(client.clone(), metrics, &obj).await?;
480            metrics.reconcile_deploy_delete_create_inc(
481                short_type_name::<K>().unwrap_or("Unknown"),
482                "immutable_api_error",
483            );
484            return self.kube_apply(client, obj, operator_name).await;
485        }
486
487        if let Err(Error::KubeError(_, cause)) = &result
488            && let kube::Error::Api(ae) = cause.as_ref()
489            && ae.code == 422
490        {
491            debug!(resource.kind = short_type_name::<K>().unwrap_or("Unknown"), resource.name = &name, resource.namespace = &namespace, reason = %ae.reason, api_message = %ae.message, "server-side apply rejected resource with a non-immutable 422; preserving the existing resource");
492        }
493
494        result
495    }
496}
497
498#[cfg(test)]
499mod test {
500    use super::{Context, is_immutable_update_message};
501
502    use crate::controller::State;
503    use crate::controller::kanidm::{KanidmKey, KanidmUser};
504    use crate::kanidm::crd::Kanidm;
505
506    use std::sync::{Arc, Mutex};
507    use std::time::Duration;
508
509    use futures::FutureExt;
510    use http::{Request, Response};
511    use kanidm_client::{KanidmClient, KanidmClientBuilder};
512    use kube::runtime::reflector::store::Writer;
513    use kube::{Client, ResourceExt, client::Body};
514    use opentelemetry::metrics::MeterProvider;
515    use tokio::io::{AsyncReadExt, AsyncWriteExt};
516    use tokio::net::TcpListener;
517
518    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
519
520    /// The `X-KANIDM-VERSION` the client expects. A debug-build client exits the process on a
521    /// mismatch, so keep this equal to the `kanidm_client` version in the workspace.
522    const KANIDM_VERSION: &str = "1.12.1";
523
524    fn test_context() -> (Context<Kanidm>, ApiServerHandle) {
525        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
526        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
527        let metrics = crate::metrics::Metrics::new(&provider.meter("test"), &["test"]);
528        let state = State::new(
529            metrics,
530            Writer::default().as_reader(),
531            Writer::default().as_reader(),
532            None,
533        );
534        (
535            state.to_context(Client::new(mock_service, "default"), "test"),
536            handle,
537        )
538    }
539
540    fn kanidm_key(kanidm: &Kanidm) -> KanidmKey {
541        KanidmKey {
542            namespace: kanidm.namespace().unwrap(),
543            name: kanidm.name_any(),
544        }
545    }
546
547    /// Start a fake Kanidm server answering every request with `status`. Returns a client
548    /// pointing to it and the request lines it received.
549    async fn start_fake_kanidm(status: u16) -> (Arc<KanidmClient>, Arc<Mutex<Vec<String>>>) {
550        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
551        let address = listener.local_addr().unwrap();
552        let requests = Arc::new(Mutex::new(Vec::new()));
553        let seen = requests.clone();
554        tokio::spawn(async move {
555            while let Ok((mut stream, _)) = listener.accept().await {
556                let mut buffer = [0u8; 4096];
557                let n = stream.read(&mut buffer).await.unwrap_or(0);
558                if let Some(line) = String::from_utf8_lossy(&buffer[..n]).lines().next() {
559                    seen.lock().unwrap().push(line.to_string());
560                }
561                let response = format!(
562                    "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\nx-kanidm-version: {KANIDM_VERSION}\r\ncontent-length: 4\r\nconnection: close\r\n\r\nnull"
563                );
564                let _ = stream.write_all(response.as_bytes()).await;
565                let _ = stream.shutdown().await;
566            }
567        });
568        let client = KanidmClientBuilder::new()
569            .no_proxy()
570            .address(format!("http://{address}"))
571            .build()
572            .unwrap();
573        (Arc::new(client), requests)
574    }
575
576    #[tokio::test]
577    async fn kanidm_cluster_client_reuses_valid_cached_session() {
578        let (ctx, mut api_server) = test_context();
579        let kanidm = Kanidm::test();
580        let (cached, requests) = start_fake_kanidm(200).await;
581        ctx.system_clients
582            .write()
583            .await
584            .insert(kanidm_key(&kanidm), cached.clone());
585
586        // Creating a new client would block on the Kubernetes API, which never answers here.
587        let client = tokio::time::timeout(
588            Duration::from_secs(5),
589            ctx.get_kanidm_cluster_client(&kanidm, KanidmUser::Admin),
590        )
591        .await
592        .expect("cached client is reused without creating a new session")
593        .expect("valid cached client");
594
595        assert!(Arc::ptr_eq(&client, &cached));
596        assert_eq!(
597            *requests.lock().unwrap(),
598            vec!["GET /v1/auth/valid HTTP/1.1".to_string()]
599        );
600        assert!(api_server.next_request().now_or_never().is_none());
601
602        ctx.release_kanidm_clients(&kanidm).await;
603        assert!(
604            ctx.system_clients
605                .read()
606                .await
607                .get(&kanidm_key(&kanidm))
608                .is_none()
609        );
610    }
611
612    #[tokio::test]
613    async fn kanidm_cluster_client_recreates_invalid_cached_session() {
614        let (ctx, mut api_server) = test_context();
615        let kanidm = Kanidm::test();
616        let (stale, requests) = start_fake_kanidm(401).await;
617        ctx.idm_clients
618            .write()
619            .await
620            .insert(kanidm_key(&kanidm), stale);
621
622        let api_server = async move {
623            let (request, send) = api_server
624                .next_request()
625                .await
626                .expect("Kanidm is read to create a new client");
627            assert_eq!(
628                request.uri().path(),
629                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/test"
630            );
631            send.send_response(
632                Response::builder()
633                    .status(404)
634                    .body(Body::from(
635                        br#"{"kind":"Status","apiVersion":"v1","status":"Failure","reason":"NotFound","code":404}"#
636                            .to_vec(),
637                    ))
638                    .unwrap(),
639            );
640        };
641        let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async {
642            tokio::join!(
643                ctx.get_kanidm_cluster_client(&kanidm, KanidmUser::IdmAdmin),
644                api_server
645            )
646        })
647        .await
648        .expect("invalid cached session falls back to client creation");
649
650        assert!(result.is_err());
651        // Validated once on the fast path and once more under the creation lock.
652        assert_eq!(
653            *requests.lock().unwrap(),
654            vec!["GET /v1/auth/valid HTTP/1.1".to_string(); 2]
655        );
656    }
657
658    #[test]
659    fn immutable_update_messages_are_classified() {
660        assert!(is_immutable_update_message(
661            "Deployment.apps \"example\" is invalid: spec.selector: Invalid value: v1.LabelSelector{}: field is immutable"
662        ));
663        assert!(is_immutable_update_message(
664            "Service \"example\" is invalid: spec.clusterIPs[0]: Invalid value: \"10.0.0.1\": may not change once set"
665        ));
666        assert!(is_immutable_update_message(
667            "StatefulSet.apps \"example\" is invalid: spec: Forbidden: updates to statefulset spec for fields other than 'replicas', 'ordinals', 'template', 'updateStrategy', 'revisionHistoryLimit', 'persistentVolumeClaimRetentionPolicy' and 'minReadySeconds' are forbidden"
668        ));
669    }
670
671    #[test]
672    fn ordinary_validation_messages_are_not_classified_as_immutable() {
673        assert!(!is_immutable_update_message(
674            "StatefulSet.apps \"example\" is invalid: spec.minReadySeconds: Invalid value: -1: must be greater than or equal to 0"
675        ));
676        assert!(!is_immutable_update_message(
677            "StatefulSet.apps \"example\" is invalid: spec.template.spec.containers[0].image: Required value"
678        ));
679    }
680}