vcon_server/tenants/
examine.rs

1//! What `check` reads of the registry: each tenant's Store examined under
2//! its own retention period, rotations left part way, and tenant removals
3//! since the master key was last rotated. Reading changes nothing.
4
5use rusqlite::OptionalExtension;
6use time::OffsetDateTime;
7use time::format_description::well_known::Rfc3339;
8
9use super::{TenantError, Tenants, io, stored_retention};
10use crate::audit::Action;
11use crate::clock::from_nanos;
12use crate::envelope::{self, Context};
13use crate::ids::TenantId;
14use crate::retention::Retention;
15use crate::store::Examination;
16
17/// One tenant as `check` found it.
18#[derive(Debug)]
19pub struct TenantExamination {
20    pub id: TenantId,
21    pub name: String,
22    /// A key rotation of this tenant was stopped part way.
23    pub rotation_stopped: bool,
24    /// Its Store examined, or why its catalog did not open.
25    pub store: Result<Examination, TenantError>,
26}
27
28/// The registry as `check` found it.
29#[derive(Debug)]
30pub struct RegistryExamination {
31    pub tenants: Vec<TenantExamination>,
32    /// Tenants removed since the master key was last rotated: their keys
33    /// are destroyed only by the next rotation.
34    pub removed_since_master: u64,
35    /// When the oldest of those removals was made.
36    pub oldest_removed_since_master: Option<OffsetDateTime>,
37}
38
39/// Whether every tenant's data is under its current key, as `serve`
40/// requires before it starts: counts of tenants only, never which.
41#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
42pub struct KeyOrder {
43    /// Tenants whose key does not unwrap under the master key.
44    pub wrapped_elsewhere: u64,
45    /// Tenants with a key rotation stopped part way.
46    pub rotations_stopped: u64,
47    /// Tenants whose catalog does not open under their key, or with a
48    /// version row naming another key.
49    pub data_under_other_keys: u64,
50}
51
52impl KeyOrder {
53    pub fn is_sound(&self) -> bool {
54        *self == KeyOrder::default()
55    }
56}
57
58/// Work that only a key rotation finishes: purges since each tenant key's
59/// rotation, and tenant removals since the master key's. Read from the
60/// audit logs only.
61#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
62pub struct RotationBacklog {
63    pub tenants_with_purges: u64,
64    pub purges: u64,
65    pub oldest_purge: Option<OffsetDateTime>,
66    pub removals: u64,
67    pub oldest_removal: Option<OffsetDateTime>,
68}
69
70impl Tenants {
71    /// Counts the tenants whose data is not all under their current key.
72    pub fn key_order(&self) -> Result<KeyOrder, TenantError> {
73        let rows: Vec<([u8; 16], Vec<u8>, bool)> = self
74            .conn()
75            .prepare(
76                "SELECT id, wrapped_tenant_key, pending_wrapped_tenant_key IS NOT NULL FROM tenant",
77            )
78            .map_err(io)?
79            .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
80            .map_err(io)?
81            .collect::<Result<_, _>>()
82            .map_err(io)?;
83        let mut order = KeyOrder::default();
84        for (id, wrapped, pending) in rows {
85            let id = TenantId(id);
86            if envelope::unwrap(&self.master, &Context::Tenant { tenant: id }, &wrapped).is_err() {
87                order.wrapped_elsewhere += 1;
88                continue;
89            }
90            if pending {
91                order.rotations_stopped += 1;
92            }
93            let elsewhere = match self.store(id) {
94                Ok(store) => store.rows_under_other_keys().map_err(io)? > 0,
95                Err(_) => true,
96            };
97            if elsewhere {
98                order.data_under_other_keys += 1;
99            }
100        }
101        Ok(order)
102    }
103
104    /// The rotation backlog: purges per tenant since its key's rotation,
105    /// and removals since the master key's.
106    pub fn rotation_backlog(&self) -> Result<RotationBacklog, TenantError> {
107        let rows: Vec<([u8; 16], i64)> = self
108            .conn()
109            .prepare("SELECT id, key_since FROM tenant")
110            .map_err(io)?
111            .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))
112            .map_err(io)?
113            .collect::<Result<_, _>>()
114            .map_err(io)?;
115        let mut backlog = RotationBacklog::default();
116        for (id, key_since) in rows {
117            let key_since = from_nanos(key_since).ok_or(TenantError::Io)?;
118            let (purges, oldest) = self
119                .store(TenantId(id))?
120                .purges_since(key_since)
121                .map_err(io)?;
122            if purges > 0 {
123                backlog.tenants_with_purges += 1;
124                backlog.purges += purges;
125                backlog.oldest_purge = earliest(backlog.oldest_purge, oldest);
126            }
127        }
128        (backlog.removals, backlog.oldest_removal) = self.removals_since_master()?;
129        Ok(backlog)
130    }
131
132    /// Tenant removals since the master key's rotation, and the oldest.
133    fn removals_since_master(&self) -> Result<(u64, Option<OffsetDateTime>), TenantError> {
134        let (since, removals) = {
135            let conn = self.conn();
136            let since: Option<i64> = conn
137                .query_row(
138                    "SELECT since FROM master_key_use WHERE master_key_id = ?1",
139                    [self.master.id().bytes()],
140                    |r| r.get(0),
141                )
142                .optional()
143                .map_err(io)?;
144            let removals: Vec<String> = conn
145                .prepare("SELECT at FROM audit WHERE action = ?1 AND status = 204")
146                .map_err(io)?
147                .query_map([Action::TenantRemove.as_str()], |r| r.get(0))
148                .map_err(io)?
149                .collect::<Result<_, _>>()
150                .map_err(io)?;
151            (since, removals)
152        };
153        let since = since
154            .map(|s| from_nanos(s).ok_or(TenantError::Io))
155            .transpose()?;
156        let (mut count, mut oldest) = (0, None);
157        for at in removals {
158            let at = OffsetDateTime::parse(&at, &Rfc3339).map_err(io)?;
159            if since.is_none_or(|since| at >= since) {
160                count += 1;
161                oldest = earliest(oldest, Some(at));
162            }
163        }
164        Ok((count, oldest))
165    }
166
167    /// Examines every tenant at `now`, under its own retention period or
168    /// `default`.
169    pub fn examine(
170        &self,
171        default: Retention,
172        now: OffsetDateTime,
173    ) -> Result<RegistryExamination, TenantError> {
174        type Row = ([u8; 16], String, Option<String>, i64, bool);
175        let rows: Vec<Row> = self
176            .conn()
177            .prepare(
178                "SELECT id, name, retention, key_since, pending_wrapped_tenant_key IS NOT NULL
179                 FROM tenant ORDER BY name",
180            )
181            .map_err(io)?
182            .query_map([], |r| {
183                Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?))
184            })
185            .map_err(io)?
186            .collect::<Result<_, _>>()
187            .map_err(io)?;
188        let time = |nanos| from_nanos(nanos).ok_or(TenantError::Io);
189        let mut tenants = Vec::new();
190        for (id, name, retention, key_since, rotation_stopped) in rows {
191            let id = TenantId(id);
192            let retention = stored_retention(retention)?.unwrap_or(default);
193            let key_since = time(key_since)?;
194            let store = self
195                .store(id)
196                .and_then(|store| store.examine(retention, key_since, now).map_err(io));
197            tenants.push(TenantExamination {
198                id,
199                name,
200                rotation_stopped,
201                store,
202            });
203        }
204        let (removed_since_master, oldest_removed_since_master) = self.removals_since_master()?;
205        let found = RegistryExamination {
206            tenants,
207            removed_since_master,
208            oldest_removed_since_master,
209        };
210        Ok(found)
211    }
212}
213
214/// The earlier of two times, either of which may be absent.
215fn earliest(a: Option<OffsetDateTime>, b: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
216    match (a, b) {
217        (Some(a), Some(b)) => Some(a.min(b)),
218        (a, b) => a.or(b),
219    }
220}
221
222#[cfg(test)]
223mod examine_tests {
224    use time::macros::datetime;
225
226    use super::super::tests::fixture;
227    use super::*;
228    use crate::audit::Actor;
229    use crate::store::Caller;
230    use crate::store::testing::vcon;
231
232    const CALLER: Caller = Caller {
233        key: crate::ids::KeyId([7; 8]),
234    };
235
236    #[test]
237    fn each_tenant_is_examined_under_its_own_period_or_the_default() {
238        let f = fixture();
239        let own = f
240            .tenants
241            .create(Actor::Cli, "own", Some(Retention::Days(1)))
242            .expect("made");
243        let default = f.tenants.create(Actor::Cli, "default", None).expect("made");
244        for id in [own, default] {
245            let store = f.tenants.store(id).expect("store");
246            let (found, body) = vcon("01a07da8-c2bb-83e5-b9a2-279e0d16bc46", "kept");
247            store.create(&CALLER, &found, &body).expect("stored");
248            store
249                .delete(&CALLER, found.uuid, Retention::Years(7))
250                .expect("deleted");
251        }
252        let found = f
253            .tenants
254            .examine(Retention::Years(7), datetime!(2026-10-30 12:00 UTC))
255            .expect("examined");
256        let by_name = |name: &str| found.tenants.iter().find(|t| t.name == name).expect(name);
257        let own = by_name("own");
258        assert_eq!(
259            own.store.as_ref().expect("opens").due,
260            1,
261            "a day has passed"
262        );
263        assert!(!own.rotation_stopped);
264        assert_eq!(by_name("default").store.as_ref().expect("opens").due, 0);
265        assert_eq!(found.tenants.len(), 2);
266    }
267
268    #[test]
269    fn a_rotation_stopped_part_way_is_reported_and_a_catalog_that_does_not_open_too() {
270        let f = fixture();
271        let id = f.tenants.create(Actor::Cli, "acme", None).expect("made");
272        *f.tenants.hooks.rekey_stop.lock().expect("hook") =
273            Some(super::super::rekey::RekeyStep::Reencrypted);
274        assert_eq!(f.tenants.rekey_tenant(Actor::Cli, id), Err(TenantError::Io));
275        let found = f
276            .tenants
277            .examine(Retention::Years(7), datetime!(2026-10-23 12:00 UTC))
278            .expect("examined");
279        let acme = &found.tenants[0];
280        assert!(acme.rotation_stopped);
281        assert!(acme.store.is_err(), "its catalog is under the pending key");
282    }
283
284    mod key_order {
285        use super::*;
286        use crate::envelope::{self, Context, Key};
287        use crate::tenants::rekey::RekeyStep;
288
289        fn with_vcons(f: &super::super::super::tests::Fixture) -> TenantId {
290            let id = f.tenants.create(Actor::Cli, "acme", None).expect("made");
291            let store = f.tenants.store(id).expect("store");
292            for n in 0..3 {
293                let (found, body) = vcon(&format!("01a07da8-c2bb-83e5-b9a2-279e0d16bc4{n}"), "k");
294                store.create(&CALLER, &found, &body).expect("stored");
295            }
296            id
297        }
298
299        #[test]
300        fn a_sound_registry_is_in_order() {
301            let f = fixture();
302            with_vcons(&f);
303            assert!(f.tenants.key_order().expect("read").is_sound());
304        }
305
306        #[test]
307        fn a_rotation_stopped_at_any_step_is_out_of_order() {
308            for (step, data) in [
309                (RekeyStep::Pending, 0),
310                (RekeyStep::Batch, 1),
311                (RekeyStep::Rewrapped, 1),
312                (RekeyStep::Reencrypted, 1),
313            ] {
314                let f = fixture();
315                let id = with_vcons(&f);
316                *f.tenants.hooks.rekey_stop.lock().expect("hook") = Some(step);
317                assert!(f.tenants.rekey_in_batches(Actor::Cli, id, 2).is_err());
318                assert_eq!(
319                    f.tenants.key_order().expect("read"),
320                    KeyOrder {
321                        wrapped_elsewhere: 0,
322                        rotations_stopped: 1,
323                        data_under_other_keys: data,
324                    },
325                    "{step:?}"
326                );
327                f.tenants.rekey_tenant(Actor::Cli, id).expect("finished");
328                assert!(f.tenants.key_order().expect("read").is_sound(), "{step:?}");
329            }
330        }
331
332        #[test]
333        fn a_tenant_key_wrapped_under_another_key_is_out_of_order() {
334            let f = fixture();
335            let id = with_vcons(&f);
336            let other = Key::generate().expect("key");
337            let foreign =
338                envelope::wrap(&other, &Context::Tenant { tenant: id }, &f.tenant_key(id));
339            f.conn()
340                .execute(
341                    "UPDATE tenant SET wrapped_tenant_key = ?1 WHERE id = ?2",
342                    rusqlite::params![foreign, id.0],
343                )
344                .expect("rewritten");
345            f.tenants.close(id);
346            assert_eq!(f.tenants.key_order().expect("read").wrapped_elsewhere, 1);
347        }
348    }
349
350    #[test]
351    fn the_rotation_backlog_counts_purges_and_removals_from_the_logs() {
352        let f = fixture();
353        let purging = f
354            .tenants
355            .create(Actor::Cli, "purging", Some(Retention::Days(0)))
356            .expect("made");
357        f.tenants.create(Actor::Cli, "quiet", None).expect("made");
358        let gone = f.tenants.create(Actor::Cli, "gone", None).expect("made");
359        let store = f.tenants.store(purging).expect("store");
360        for (n, at) in [
361            (0, datetime!(2026-10-24 10:00 UTC)),
362            (1, datetime!(2026-10-24 11:00 UTC)),
363        ] {
364            f.clock.set(at);
365            let (found, body) = vcon(&format!("01a07da8-c2bb-83e5-b9a2-279e0d16bc4{n}"), "k");
366            store.create(&CALLER, &found, &body).expect("stored");
367            store
368                .delete(&CALLER, found.uuid, Retention::Days(0))
369                .expect("deleted and purged");
370        }
371        drop(store);
372        f.clock.set(datetime!(2026-10-24 12:00 UTC));
373        f.tenants
374            .remove(Actor::Cli, gone, crate::tenants::Removal::Empty)
375            .expect("removed");
376        assert_eq!(
377            f.tenants.rotation_backlog().expect("read"),
378            RotationBacklog {
379                tenants_with_purges: 1,
380                purges: 2,
381                oldest_purge: Some(datetime!(2026-10-24 10:00 UTC)),
382                removals: 1,
383                oldest_removal: Some(datetime!(2026-10-24 12:00 UTC)),
384            }
385        );
386    }
387
388    #[test]
389    fn removals_since_the_master_key_was_rotated_are_counted() {
390        let f = fixture();
391        let gone = f.tenants.create(Actor::Cli, "gone", None).expect("made");
392        f.tenants.create(Actor::Cli, "kept", None).expect("made");
393        f.clock.set(datetime!(2026-10-24 08:00 UTC));
394        f.tenants
395            .remove(Actor::Cli, gone, crate::tenants::Removal::Empty)
396            .expect("removed");
397        // A removal from before the master key's rotation, and a refused one.
398        f.conn()
399            .execute_batch(
400                "INSERT INTO audit (at, actor, action, target, version, status, count)
401                 VALUES ('2026-10-01T00:00:00Z', 'cli', 'tenant_remove', NULL, NULL, 204, 1),
402                        ('2026-10-24T09:00:00Z', 'cli', 'tenant_remove', NULL, NULL, 409, 1);",
403            )
404            .expect("older records");
405        let found = f
406            .tenants
407            .examine(Retention::Years(7), datetime!(2026-10-25 12:00 UTC))
408            .expect("examined");
409        assert_eq!(found.removed_since_master, 1);
410        assert_eq!(
411            found.oldest_removed_since_master,
412            Some(datetime!(2026-10-24 08:00 UTC))
413        );
414        assert_eq!(found.tenants.len(), 1);
415    }
416}
vcon_server/tenants/examine.rs

vcon_server/tenants/
examine.rs
