kube_runtime/
finalizer.rs

1//! Finalizer helper for [`Controller`](crate::Controller) reconcilers
2use crate::controller::Action;
3use futures::{TryFuture, TryFutureExt};
4use json_patch::{jsonptr::PointerBuf, AddOperation, PatchOperation, RemoveOperation, TestOperation};
5use kube_client::{
6    api::{Patch, PatchParams},
7    Api, Resource, ResourceExt,
8};
9
10use serde::{de::DeserializeOwned, Serialize};
11use std::{error::Error as StdError, fmt::Debug, str::FromStr, sync::Arc};
12use thiserror::Error;
13
14#[derive(Debug, Error)]
15pub enum Error<ReconcileErr>
16where
17    ReconcileErr: StdError + 'static,
18{
19    #[error("failed to apply object: {0}")]
20    ApplyFailed(#[source] ReconcileErr),
21    #[error("failed to clean up object: {0}")]
22    CleanupFailed(#[source] ReconcileErr),
23    #[error("failed to add finalizer: {0}")]
24    AddFinalizer(#[source] kube_client::Error),
25    #[error("failed to remove finalizer: {0}")]
26    RemoveFinalizer(#[source] kube_client::Error),
27    #[error("object has no name")]
28    UnnamedObject,
29    #[error("invalid finalizer")]
30    InvalidFinalizer,
31}
32
33struct FinalizerState {
34    finalizer_index: Option<usize>,
35    is_deleting: bool,
36}
37
38impl FinalizerState {
39    fn for_object<K: Resource>(obj: &K, finalizer_name: &str) -> Self {
40        Self {
41            finalizer_index: obj
42                .finalizers()
43                .iter()
44                .enumerate()
45                .find(|(_, fin)| *fin == finalizer_name)
46                .map(|(i, _)| i),
47            is_deleting: obj.meta().deletion_timestamp.is_some(),
48        }
49    }
50}
51
52/// Reconcile an object in a way that requires cleanup before an object can be deleted.
53///
54/// It does this by managing a [`ObjectMeta::finalizers`] entry,
55/// which prevents the object from being deleted before the cleanup is done.
56///
57/// In typical usage, if you use `finalizer` then it should be the only top-level "action"
58/// in your [`applier`](crate::applier)/[`Controller`](crate::Controller)'s `reconcile` function.
59///
60/// # Expected Flow
61///
62/// 1. User creates object
63/// 2. Reconciler sees object
64/// 3. `finalizer` adds `finalizer_name` to [`ObjectMeta::finalizers`]
65/// 4. Reconciler sees updated object
66/// 5. `finalizer` runs [`Event::Apply`]
67/// 6. User updates object
68/// 7. Reconciler sees updated object
69/// 8. `finalizer` runs [`Event::Apply`]
70/// 9. User deletes object
71/// 10. Reconciler sees deleting object
72/// 11. `finalizer` runs [`Event::Cleanup`]
73/// 12. `finalizer` removes `finalizer_name` from [`ObjectMeta::finalizers`]
74/// 13. Kubernetes sees that all [`ObjectMeta::finalizers`] are gone and finally deletes the object
75///
76/// # Guarantees
77///
78/// If [`Event::Apply`] is ever started then [`Event::Cleanup`] must succeed before the Kubernetes object deletion completes.
79///
80/// # Assumptions
81///
82/// `finalizer_name` must be unique among the controllers interacting with the object
83///
84/// [`Event::Apply`] and [`Event::Cleanup`] must both be idempotent, and tolerate being executed several times (even if previously cancelled).
85///
86/// [`Event::Cleanup`] must tolerate [`Event::Apply`] never having ran at all, or never having succeeded. Keep in mind that
87/// even infallible `.await`s are cancellation points.
88///
89/// # Caveats
90///
91/// Object deletes will get stuck while the controller is not running, or if `cleanup` fails for some reason.
92///
93/// `reconcile` should take the object that the [`Event`] contains, rather than trying to reuse `obj`, since it may have been updated.
94///
95/// # Errors
96///
97/// [`Event::Apply`] and [`Event::Cleanup`] are both fallible, their errors are passed through as [`Error::ApplyFailed`]
98/// and [`Error::CleanupFailed`], respectively.
99///
100/// In addition, adding and removing the finalizer itself may fail. In particular, this may be because of
101/// network errors, lacking permissions, or because another `finalizer` was updated in the meantime on the same object.
102///
103/// [`ObjectMeta::finalizers`]: kube_client::api::ObjectMeta#structfield.finalizers
104pub async fn finalizer<K, ReconcileFut>(
105    api: &Api<K>,
106    finalizer_name: &str,
107    obj: Arc<K>,
108    reconcile: impl FnOnce(Event<K>) -> ReconcileFut,
109) -> Result<Action, Error<ReconcileFut::Error>>
110where
111    K: Resource + Clone + DeserializeOwned + Serialize + Debug,
112    ReconcileFut: TryFuture<Ok = Action>,
113    ReconcileFut::Error: StdError + 'static,
114{
115    match FinalizerState::for_object(&*obj, finalizer_name) {
116        FinalizerState {
117            finalizer_index: Some(_),
118            is_deleting: false,
119        } => TryFutureExt::into_future(reconcile(Event::Apply(obj)))
120            .await
121            .map_err(Error::ApplyFailed),
122        FinalizerState {
123            finalizer_index: Some(finalizer_i),
124            is_deleting: true,
125        } => {
126            // Cleanup reconciliation must succeed before it's safe to remove the finalizer
127            let name = obj.meta().name.clone().ok_or(Error::UnnamedObject)?;
128            let action = TryFutureExt::into_future(reconcile(Event::Cleanup(obj)))
129                .await
130                // Short-circuit, so that we keep the finalizer if cleanup fails
131                .map_err(Error::CleanupFailed)?;
132            // Cleanup was successful, remove the finalizer so that deletion can continue
133            let finalizer_path = format!("/metadata/finalizers/{finalizer_i}");
134            api.patch::<K>(
135                &name,
136                &PatchParams::default(),
137                &Patch::Json(json_patch::Patch(vec![
138                    // All finalizers run concurrently and we use an integer index
139                    // `Test` ensures that we fail instead of deleting someone else's finalizer
140                    // (in which case a new `Cleanup` event will be sent)
141                    PatchOperation::Test(TestOperation {
142                        path: PointerBuf::from_str(finalizer_path.as_str())
143                            .map_err(|_err| Error::InvalidFinalizer)?,
144                        value: finalizer_name.into(),
145                    }),
146                    PatchOperation::Remove(RemoveOperation {
147                        path: PointerBuf::from_str(finalizer_path.as_str())
148                            .map_err(|_err| Error::InvalidFinalizer)?,
149                    }),
150                ])),
151            )
152            .await
153            .map_err(Error::RemoveFinalizer)?;
154            Ok(action)
155        }
156        FinalizerState {
157            finalizer_index: None,
158            is_deleting: false,
159        } => {
160            // Finalizer must be added before it's safe to run an `Apply` reconciliation
161            let patch = json_patch::Patch(if obj.finalizers().is_empty() {
162                vec![
163                    PatchOperation::Test(TestOperation {
164                        path: PointerBuf::from_str("/metadata/finalizers")
165                            .map_err(|_err| Error::InvalidFinalizer)?,
166                        value: serde_json::Value::Null,
167                    }),
168                    PatchOperation::Add(AddOperation {
169                        path: PointerBuf::from_str("/metadata/finalizers")
170                            .map_err(|_err| Error::InvalidFinalizer)?,
171                        value: vec![finalizer_name].into(),
172                    }),
173                ]
174            } else {
175                vec![
176                    // Kubernetes doesn't automatically deduplicate finalizers (see
177                    // https://github.com/kube-rs/kube/issues/964#issuecomment-1197311254),
178                    // so we need to fail and retry if anyone else has added the finalizer in the meantime
179                    PatchOperation::Test(TestOperation {
180                        path: PointerBuf::from_str("/metadata/finalizers")
181                            .map_err(|_err| Error::InvalidFinalizer)?,
182                        value: obj.finalizers().into(),
183                    }),
184                    PatchOperation::Add(AddOperation {
185                        path: PointerBuf::from_str("/metadata/finalizers/-")
186                            .map_err(|_err| Error::InvalidFinalizer)?,
187                        value: finalizer_name.into(),
188                    }),
189                ]
190            });
191            api.patch::<K>(
192                obj.meta().name.as_deref().ok_or(Error::UnnamedObject)?,
193                &PatchParams::default(),
194                &Patch::Json(patch),
195            )
196            .await
197            .map_err(Error::AddFinalizer)?;
198            // No point applying here, since the patch will cause a new reconciliation
199            Ok(Action::await_change())
200        }
201        FinalizerState {
202            finalizer_index: None,
203            is_deleting: true,
204        } => {
205            // Our work here is done
206            Ok(Action::await_change())
207        }
208    }
209}
210
211/// A representation of an action that should be taken by a reconciler.
212pub enum Event<K> {
213    /// The reconciler should ensure that the actual state matches the state desired in the object.
214    ///
215    /// This must be idempotent, since it may be recalled if, for example (this list is non-exhaustive):
216    ///
217    /// - The controller is restarted
218    /// - The object is updated
219    /// - The reconciliation fails
220    /// - The grinch attacks
221    Apply(Arc<K>),
222    /// The object is being deleted, and the reconciler should remove all resources that it owns.
223    ///
224    /// This must be idempotent, since it may be recalled if, for example (this list is non-exhaustive):
225    ///
226    /// - The controller is restarted while the deletion is in progress
227    /// - The reconciliation fails
228    /// - Another finalizer was removed in the meantime
229    /// - The grinch's heart grows a size or two
230    Cleanup(Arc<K>),
231}