kube_runtime/finalizer.rs
1//! Finalizer helper for [`Controller`](crate::Controller) reconcilers
2use crate::controller::Action;
3use futures::{TryFuture, TryFutureExt};
4use json_patch::{jsonptr::PointerBuf, AddOperation, PatchOperation, RemoveOperation, TestOperation};
5use kube_client::{
6 api::{Patch, PatchParams},
7 Api, Resource, ResourceExt,
8};
9
10use serde::{de::DeserializeOwned, Serialize};
11use std::{error::Error as StdError, fmt::Debug, str::FromStr, sync::Arc};
12use thiserror::Error;
13
14#[derive(Debug, Error)]
15pub enum Error<ReconcileErr>
16where
17 ReconcileErr: StdError + 'static,
18{
19 #[error("failed to apply object: {0}")]
20 ApplyFailed(#[source] ReconcileErr),
21 #[error("failed to clean up object: {0}")]
22 CleanupFailed(#[source] ReconcileErr),
23 #[error("failed to add finalizer: {0}")]
24 AddFinalizer(#[source] kube_client::Error),
25 #[error("failed to remove finalizer: {0}")]
26 RemoveFinalizer(#[source] kube_client::Error),
27 #[error("object has no name")]
28 UnnamedObject,
29 #[error("invalid finalizer")]
30 InvalidFinalizer,
31}
32
33struct FinalizerState {
34 finalizer_index: Option<usize>,
35 is_deleting: bool,
36}
37
38impl FinalizerState {
39 fn for_object<K: Resource>(obj: &K, finalizer_name: &str) -> Self {
40 Self {
41 finalizer_index: obj
42 .finalizers()
43 .iter()
44 .enumerate()
45 .find(|(_, fin)| *fin == finalizer_name)
46 .map(|(i, _)| i),
47 is_deleting: obj.meta().deletion_timestamp.is_some(),
48 }
49 }
50}
51
52/// Reconcile an object in a way that requires cleanup before an object can be deleted.
53///
54/// It does this by managing a [`ObjectMeta::finalizers`] entry,
55/// which prevents the object from being deleted before the cleanup is done.
56///
57/// In typical usage, if you use `finalizer` then it should be the only top-level "action"
58/// in your [`applier`](crate::applier)/[`Controller`](crate::Controller)'s `reconcile` function.
59///
60/// # Expected Flow
61///
62/// 1. User creates object
63/// 2. Reconciler sees object
64/// 3. `finalizer` adds `finalizer_name` to [`ObjectMeta::finalizers`]
65/// 4. Reconciler sees updated object
66/// 5. `finalizer` runs [`Event::Apply`]
67/// 6. User updates object
68/// 7. Reconciler sees updated object
69/// 8. `finalizer` runs [`Event::Apply`]
70/// 9. User deletes object
71/// 10. Reconciler sees deleting object
72/// 11. `finalizer` runs [`Event::Cleanup`]
73/// 12. `finalizer` removes `finalizer_name` from [`ObjectMeta::finalizers`]
74/// 13. Kubernetes sees that all [`ObjectMeta::finalizers`] are gone and finally deletes the object
75///
76/// # Guarantees
77///
78/// If [`Event::Apply`] is ever started then [`Event::Cleanup`] must succeed before the Kubernetes object deletion completes.
79///
80/// # Assumptions
81///
82/// `finalizer_name` must be unique among the controllers interacting with the object
83///
84/// [`Event::Apply`] and [`Event::Cleanup`] must both be idempotent, and tolerate being executed several times (even if previously cancelled).
85///
86/// [`Event::Cleanup`] must tolerate [`Event::Apply`] never having ran at all, or never having succeeded. Keep in mind that
87/// even infallible `.await`s are cancellation points.
88///
89/// # Caveats
90///
91/// Object deletes will get stuck while the controller is not running, or if `cleanup` fails for some reason.
92///
93/// `reconcile` should take the object that the [`Event`] contains, rather than trying to reuse `obj`, since it may have been updated.
94///
95/// # Errors
96///
97/// [`Event::Apply`] and [`Event::Cleanup`] are both fallible, their errors are passed through as [`Error::ApplyFailed`]
98/// and [`Error::CleanupFailed`], respectively.
99///
100/// In addition, adding and removing the finalizer itself may fail. In particular, this may be because of
101/// network errors, lacking permissions, or because another `finalizer` was updated in the meantime on the same object.
102///
103/// [`ObjectMeta::finalizers`]: kube_client::api::ObjectMeta#structfield.finalizers
104pub async fn finalizer<K, ReconcileFut>(
105 api: &Api<K>,
106 finalizer_name: &str,
107 obj: Arc<K>,
108 reconcile: impl FnOnce(Event<K>) -> ReconcileFut,
109) -> Result<Action, Error<ReconcileFut::Error>>
110where
111 K: Resource + Clone + DeserializeOwned + Serialize + Debug,
112 ReconcileFut: TryFuture<Ok = Action>,
113 ReconcileFut::Error: StdError + 'static,
114{
115 match FinalizerState::for_object(&*obj, finalizer_name) {
116 FinalizerState {
117 finalizer_index: Some(_),
118 is_deleting: false,
119 } => TryFutureExt::into_future(reconcile(Event::Apply(obj)))
120 .await
121 .map_err(Error::ApplyFailed),
122 FinalizerState {
123 finalizer_index: Some(finalizer_i),
124 is_deleting: true,
125 } => {
126 // Cleanup reconciliation must succeed before it's safe to remove the finalizer
127 let name = obj.meta().name.clone().ok_or(Error::UnnamedObject)?;
128 let action = TryFutureExt::into_future(reconcile(Event::Cleanup(obj)))
129 .await
130 // Short-circuit, so that we keep the finalizer if cleanup fails
131 .map_err(Error::CleanupFailed)?;
132 // Cleanup was successful, remove the finalizer so that deletion can continue
133 let finalizer_path = format!("/metadata/finalizers/{finalizer_i}");
134 api.patch::<K>(
135 &name,
136 &PatchParams::default(),
137 &Patch::Json(json_patch::Patch(vec![
138 // All finalizers run concurrently and we use an integer index
139 // `Test` ensures that we fail instead of deleting someone else's finalizer
140 // (in which case a new `Cleanup` event will be sent)
141 PatchOperation::Test(TestOperation {
142 path: PointerBuf::from_str(finalizer_path.as_str())
143 .map_err(|_err| Error::InvalidFinalizer)?,
144 value: finalizer_name.into(),
145 }),
146 PatchOperation::Remove(RemoveOperation {
147 path: PointerBuf::from_str(finalizer_path.as_str())
148 .map_err(|_err| Error::InvalidFinalizer)?,
149 }),
150 ])),
151 )
152 .await
153 .map_err(Error::RemoveFinalizer)?;
154 Ok(action)
155 }
156 FinalizerState {
157 finalizer_index: None,
158 is_deleting: false,
159 } => {
160 // Finalizer must be added before it's safe to run an `Apply` reconciliation
161 let patch = json_patch::Patch(if obj.finalizers().is_empty() {
162 vec![
163 PatchOperation::Test(TestOperation {
164 path: PointerBuf::from_str("/metadata/finalizers")
165 .map_err(|_err| Error::InvalidFinalizer)?,
166 value: serde_json::Value::Null,
167 }),
168 PatchOperation::Add(AddOperation {
169 path: PointerBuf::from_str("/metadata/finalizers")
170 .map_err(|_err| Error::InvalidFinalizer)?,
171 value: vec![finalizer_name].into(),
172 }),
173 ]
174 } else {
175 vec![
176 // Kubernetes doesn't automatically deduplicate finalizers (see
177 // https://github.com/kube-rs/kube/issues/964#issuecomment-1197311254),
178 // so we need to fail and retry if anyone else has added the finalizer in the meantime
179 PatchOperation::Test(TestOperation {
180 path: PointerBuf::from_str("/metadata/finalizers")
181 .map_err(|_err| Error::InvalidFinalizer)?,
182 value: obj.finalizers().into(),
183 }),
184 PatchOperation::Add(AddOperation {
185 path: PointerBuf::from_str("/metadata/finalizers/-")
186 .map_err(|_err| Error::InvalidFinalizer)?,
187 value: finalizer_name.into(),
188 }),
189 ]
190 });
191 api.patch::<K>(
192 obj.meta().name.as_deref().ok_or(Error::UnnamedObject)?,
193 &PatchParams::default(),
194 &Patch::Json(patch),
195 )
196 .await
197 .map_err(Error::AddFinalizer)?;
198 // No point applying here, since the patch will cause a new reconciliation
199 Ok(Action::await_change())
200 }
201 FinalizerState {
202 finalizer_index: None,
203 is_deleting: true,
204 } => {
205 // Our work here is done
206 Ok(Action::await_change())
207 }
208 }
209}
210
211/// A representation of an action that should be taken by a reconciler.
212pub enum Event<K> {
213 /// The reconciler should ensure that the actual state matches the state desired in the object.
214 ///
215 /// This must be idempotent, since it may be recalled if, for example (this list is non-exhaustive):
216 ///
217 /// - The controller is restarted
218 /// - The object is updated
219 /// - The reconciliation fails
220 /// - The grinch attacks
221 Apply(Arc<K>),
222 /// The object is being deleted, and the reconciler should remove all resources that it owns.
223 ///
224 /// This must be idempotent, since it may be recalled if, for example (this list is non-exhaustive):
225 ///
226 /// - The controller is restarted while the deletion is in progress
227 /// - The reconciliation fails
228 /// - Another finalizer was removed in the meantime
229 /// - The grinch's heart grows a size or two
230 Cleanup(Arc<K>),
231}