vcon_server/tenants/
examine.rs

1//! What `check` reads of the registry: each tenant's Store examined under
2//! its own retention period, rotations left part way, and tenant removals
3//! since the master key was last rotated. Reading changes nothing.
4
5use time::OffsetDateTime;
6use time::format_description::well_known::Rfc3339;
7
8use super::{OpenFailure, TenantError, Tenants, io, stored_retention};
9use crate::audit::Action;
10use crate::clock::from_nanos;
11use crate::envelope::{self, Context};
12use crate::ids::TenantId;
13use crate::retention::Retention;
14use crate::store::Examination;
15
16/// One tenant as `check` found it.
17#[derive(Debug)]
18pub struct TenantExamination {
19    pub id: TenantId,
20    pub name: String,
21    /// A key rotation of this tenant was stopped part way.
22    pub rotation_stopped: bool,
23    /// Its Store examined, or why its catalog did not open.
24    pub store: Result<Examination, OpenFailure>,
25}
26
27/// The registry as `check` found it.
28#[derive(Debug)]
29pub struct RegistryExamination {
30    pub tenants: Vec<TenantExamination>,
31    /// Tenants removed since the master key was last rotated: their keys
32    /// are destroyed only by the next rotation.
33    pub removed_since_master: u64,
34    /// When the oldest of those removals was made.
35    pub oldest_removed_since_master: Option<OffsetDateTime>,
36}
37
38/// Whether every tenant's data is under its current key, as `serve`
39/// requires before it starts: counts of tenants only, never which, and no
40/// path, since a path names a tenant.
41#[derive(Debug, Clone, PartialEq, Eq, Default)]
42pub struct KeyOrder {
43    /// Tenants whose key does not unwrap under the master key.
44    pub wrapped_elsewhere: u64,
45    /// Tenants with a key rotation stopped part way.
46    pub rotations_stopped: u64,
47    /// Tenants whose catalog does not open under their key, or with a
48    /// version row naming another key.
49    pub data_under_other_keys: u64,
50    /// Tenants whose Store did not open for a reason other than its key,
51    /// such as files this process may not use.
52    pub catalogs_unopened: u64,
53    /// The system's error for the first of those that has one, such as
54    /// `Permission denied (os error 13)`.
55    pub unopened_error: Option<String>,
56}
57
58impl KeyOrder {
59    pub fn is_sound(&self) -> bool {
60        *self == KeyOrder::default()
61    }
62}
63
64/// Work that only a key rotation finishes: purges since each tenant key's
65/// rotation, and tenant removals since the master key's. Read from the
66/// audit logs only.
67#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
68pub struct RotationBacklog {
69    pub tenants_with_purges: u64,
70    pub purges: u64,
71    pub oldest_purge: Option<OffsetDateTime>,
72    pub removals: u64,
73    pub oldest_removal: Option<OffsetDateTime>,
74}
75
76impl Tenants {
77    /// Counts the tenants whose data is not all under their current key.
78    pub fn key_order(&self) -> Result<KeyOrder, TenantError> {
79        let rows = self.registry.snapshot()?.tenants()?;
80        let mut order = KeyOrder::default();
81        for row in rows {
82            let (id, wrapped, pending) = (row.id, row.wrapped, row.pending.is_some());
83            if envelope::unwrap(&self.master, &Context::Tenant { tenant: id }, &wrapped).is_err() {
84                order.wrapped_elsewhere += 1;
85                continue;
86            }
87            if pending {
88                order.rotations_stopped += 1;
89            }
90            let elsewhere = match self.store_or_why(id) {
91                Ok(store) => store.rows_under_other_keys().map_err(io)? > 0,
92                Err(OpenFailure::WrongKey) => true,
93                Err(failure) => {
94                    order.catalogs_unopened += 1;
95                    if let OpenFailure::Access { error, .. } = failure {
96                        order.unopened_error.get_or_insert(error.to_string());
97                    }
98                    false
99                }
100            };
101            if elsewhere {
102                order.data_under_other_keys += 1;
103            }
104        }
105        Ok(order)
106    }
107
108    /// The rotation backlog: purges per tenant since its key's rotation,
109    /// and removals since the master key's.
110    pub fn rotation_backlog(&self) -> Result<RotationBacklog, TenantError> {
111        let rows = self.registry.snapshot()?.tenants()?;
112        let mut backlog = RotationBacklog::default();
113        for row in rows {
114            let key_since = from_nanos(row.key_since).ok_or(TenantError::Io)?;
115            let (purges, oldest) = self.store(row.id)?.purges_since(key_since).map_err(io)?;
116            if purges > 0 {
117                backlog.tenants_with_purges += 1;
118                backlog.purges += purges;
119                backlog.oldest_purge = earliest(backlog.oldest_purge, oldest);
120            }
121        }
122        (backlog.removals, backlog.oldest_removal) = self.removals_since_master()?;
123        Ok(backlog)
124    }
125
126    /// Tenant removals since the master key's rotation, and the oldest.
127    fn removals_since_master(&self) -> Result<(u64, Option<OffsetDateTime>), TenantError> {
128        let (since, removals) = {
129            let mut tx = self.registry.snapshot()?;
130            let since = tx.master_since(self.master.id())?;
131            let removals = tx.record_times(Action::TenantRemove, 204)?;
132            (since, removals)
133        };
134        let since = since
135            .map(|s| from_nanos(s).ok_or(TenantError::Io))
136            .transpose()?;
137        let (mut count, mut oldest) = (0, None);
138        for at in removals {
139            let at = OffsetDateTime::parse(&at, &Rfc3339).map_err(io)?;
140            if since.is_none_or(|since| at >= since) {
141                count += 1;
142                oldest = earliest(oldest, Some(at));
143            }
144        }
145        Ok((count, oldest))
146    }
147
148    /// Examines every tenant at `now`, under its own retention period or
149    /// `default`.
150    pub fn examine(
151        &self,
152        default: Retention,
153        now: OffsetDateTime,
154    ) -> Result<RegistryExamination, TenantError> {
155        let rows = self.registry.snapshot()?.tenants()?;
156        let time = |nanos| from_nanos(nanos).ok_or(TenantError::Io);
157        let mut tenants = Vec::new();
158        for row in rows {
159            let (id, name, retention, key_since) = (row.id, row.name, row.retention, row.key_since);
160            let rotation_stopped = row.pending.is_some();
161            let retention = super::kept_for(stored_retention(retention)?, default);
162            let key_since = time(key_since)?;
163            let store = self.store_or_why(id).and_then(|store| {
164                store
165                    .examine(retention, key_since, now)
166                    .map_err(|e| OpenFailure::Refused(io(e)))
167            });
168            tenants.push(TenantExamination {
169                id,
170                name,
171                rotation_stopped,
172                store,
173            });
174        }
175        let (removed_since_master, oldest_removed_since_master) = self.removals_since_master()?;
176        let found = RegistryExamination {
177            tenants,
178            removed_since_master,
179            oldest_removed_since_master,
180        };
181        Ok(found)
182    }
183}
184
185/// The earlier of two times, either of which may be absent.
186fn earliest(a: Option<OffsetDateTime>, b: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
187    match (a, b) {
188        (Some(a), Some(b)) => Some(a.min(b)),
189        (a, b) => a.or(b),
190    }
191}
192
193#[cfg(test)]
194mod examine_tests {
195    use time::macros::datetime;
196
197    use super::super::tests::fixture;
198    use super::*;
199    use crate::audit::Actor;
200    use crate::store::Caller;
201    use crate::store::testing::vcon;
202
203    const CALLER: Caller = Caller {
204        key: crate::ids::KeyId([7; 8]),
205    };
206
207    #[test]
208    fn each_tenant_is_examined_under_its_own_period_or_the_default() {
209        let f = fixture();
210        let own = f
211            .tenants
212            .create(Actor::Cli, "own", Some(Retention::Days(1)))
213            .expect("made");
214        let default = f.tenants.create(Actor::Cli, "default", None).expect("made");
215        for id in [own, default] {
216            let store = f.tenants.store(id).expect("store");
217            let (found, body) = vcon("01a07da8-c2bb-83e5-b9a2-279e0d16bc46", "kept");
218            store.create(&CALLER, &found, &body).expect("stored");
219            store
220                .delete(&CALLER, found.uuid, Retention::Years(7))
221                .expect("deleted");
222        }
223        let found = f
224            .tenants
225            .examine(Retention::Years(7), datetime!(2026-10-30 12:00 UTC))
226            .expect("examined");
227        let by_name = |name: &str| found.tenants.iter().find(|t| t.name == name).expect(name);
228        let own = by_name("own");
229        assert_eq!(
230            own.store.as_ref().expect("opens").due,
231            1,
232            "a day has passed"
233        );
234        assert!(!own.rotation_stopped);
235        assert_eq!(by_name("default").store.as_ref().expect("opens").due, 0);
236        assert_eq!(found.tenants.len(), 2);
237    }
238
239    #[test]
240    fn a_rotation_stopped_part_way_is_reported_and_a_catalog_that_does_not_open_too() {
241        let f = fixture();
242        let id = f.tenants.create(Actor::Cli, "acme", None).expect("made");
243        *f.tenants.hooks.rekey_stop.lock().expect("hook") =
244            Some(super::super::rekey::RekeyStep::Reencrypted);
245        assert_eq!(f.tenants.rekey_tenant(Actor::Cli, id), Err(TenantError::Io));
246        let found = f
247            .tenants
248            .examine(Retention::Years(7), datetime!(2026-10-23 12:00 UTC))
249            .expect("examined");
250        let acme = &found.tenants[0];
251        assert!(acme.rotation_stopped);
252        assert!(
253            matches!(acme.store, Err(OpenFailure::WrongKey)),
254            "its catalog is under the pending key: {:?}",
255            acme.store
256        );
257    }
258
259    /// A tenant's files this process may not write, such as ones a command
260    /// run as root left, are reported as that, with the path and the
261    /// system's error; only a key that does not open the catalog is a key
262    /// problem.
263    #[test]
264    fn a_catalog_the_process_may_not_open_is_an_access_failure_not_a_key_one() {
265        use std::os::unix::fs::PermissionsExt;
266        let f = fixture();
267        // Made, but not yet opened through the registry, as at start-up.
268        let id = f.tenants.create(Actor::Cli, "acme", None).expect("made");
269        let dir = f.dir.path().join("tenants").join(id.to_string());
270        let mode =
271            |m| std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(m)).expect("chmod");
272        mode(0o500);
273        let order = f.tenants.key_order();
274        let found = f
275            .tenants
276            .examine(Retention::Years(7), datetime!(2026-10-23 12:00 UTC));
277        mode(0o700);
278        let order = order.expect("read");
279        assert_eq!(order.data_under_other_keys, 0, "{order:?}");
280        assert_eq!(order.catalogs_unopened, 1, "{order:?}");
281        assert_eq!(
282            order.unopened_error.as_deref(),
283            Some("Permission denied (os error 13)")
284        );
285        assert!(!order.is_sound());
286        match &found.expect("examined").tenants[0].store {
287            Err(OpenFailure::Access { path, error }) => {
288                assert_eq!(path, &dir);
289                assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied);
290            }
291            other => panic!("{other:?}"),
292        }
293        assert!(
294            f.tenants.key_order().expect("read").is_sound(),
295            "once allowed"
296        );
297    }
298
299    mod key_order {
300        use super::*;
301        use crate::envelope::{self, Context, Key};
302        use crate::tenants::rekey::RekeyStep;
303
304        fn with_vcons(f: &super::super::super::tests::Fixture) -> TenantId {
305            let id = f.tenants.create(Actor::Cli, "acme", None).expect("made");
306            let store = f.tenants.store(id).expect("store");
307            for n in 0..3 {
308                let (found, body) = vcon(&format!("01a07da8-c2bb-83e5-b9a2-279e0d16bc4{n}"), "k");
309                store.create(&CALLER, &found, &body).expect("stored");
310            }
311            id
312        }
313
314        #[test]
315        fn a_sound_registry_is_in_order() {
316            let f = fixture();
317            with_vcons(&f);
318            assert!(f.tenants.key_order().expect("read").is_sound());
319        }
320
321        #[test]
322        fn a_rotation_stopped_at_any_step_is_out_of_order() {
323            for (step, data) in [
324                (RekeyStep::Pending, 0),
325                (RekeyStep::Batch, 1),
326                (RekeyStep::Rewrapped, 1),
327                (RekeyStep::Reencrypted, 1),
328            ] {
329                let f = fixture();
330                let id = with_vcons(&f);
331                *f.tenants.hooks.rekey_stop.lock().expect("hook") = Some(step);
332                assert!(f.tenants.rekey_in_batches(Actor::Cli, id, 2).is_err());
333                assert_eq!(
334                    f.tenants.key_order().expect("read"),
335                    KeyOrder {
336                        wrapped_elsewhere: 0,
337                        rotations_stopped: 1,
338                        data_under_other_keys: data,
339                        ..KeyOrder::default()
340                    },
341                    "{step:?}"
342                );
343                f.tenants.rekey_tenant(Actor::Cli, id).expect("finished");
344                assert!(f.tenants.key_order().expect("read").is_sound(), "{step:?}");
345            }
346        }
347
348        #[test]
349        fn a_tenant_key_wrapped_under_another_key_is_out_of_order() {
350            let f = fixture();
351            let id = with_vcons(&f);
352            let other = Key::generate().expect("key");
353            let foreign =
354                envelope::wrap(&other, &Context::Tenant { tenant: id }, &f.tenant_key(id));
355            f.conn()
356                .execute(
357                    "UPDATE tenant SET wrapped_tenant_key = ?1 WHERE id = ?2",
358                    rusqlite::params![foreign, id.0],
359                )
360                .expect("rewritten");
361            f.tenants.close(id);
362            assert_eq!(f.tenants.key_order().expect("read").wrapped_elsewhere, 1);
363        }
364    }
365
366    #[test]
367    fn the_rotation_backlog_counts_purges_and_removals_from_the_logs() {
368        let f = fixture();
369        let purging = f
370            .tenants
371            .create(Actor::Cli, "purging", Some(Retention::Days(0)))
372            .expect("made");
373        f.tenants.create(Actor::Cli, "quiet", None).expect("made");
374        let gone = f.tenants.create(Actor::Cli, "gone", None).expect("made");
375        let store = f.tenants.store(purging).expect("store");
376        for (n, at) in [
377            (0, datetime!(2026-10-24 10:00 UTC)),
378            (1, datetime!(2026-10-24 11:00 UTC)),
379        ] {
380            f.clock.set(at);
381            let (found, body) = vcon(&format!("01a07da8-c2bb-83e5-b9a2-279e0d16bc4{n}"), "k");
382            store.create(&CALLER, &found, &body).expect("stored");
383            store
384                .delete(&CALLER, found.uuid, Retention::Days(0))
385                .expect("deleted and purged");
386        }
387        drop(store);
388        f.clock.set(datetime!(2026-10-24 12:00 UTC));
389        f.tenants
390            .remove(Actor::Cli, gone, crate::tenants::Removal::Empty)
391            .expect("removed");
392        assert_eq!(
393            f.tenants.rotation_backlog().expect("read"),
394            RotationBacklog {
395                tenants_with_purges: 1,
396                purges: 2,
397                oldest_purge: Some(datetime!(2026-10-24 10:00 UTC)),
398                removals: 1,
399                oldest_removal: Some(datetime!(2026-10-24 12:00 UTC)),
400            }
401        );
402    }
403
404    #[test]
405    fn removals_since_the_master_key_was_rotated_are_counted() {
406        let f = fixture();
407        let gone = f.tenants.create(Actor::Cli, "gone", None).expect("made");
408        f.tenants.create(Actor::Cli, "kept", None).expect("made");
409        f.clock.set(datetime!(2026-10-24 08:00 UTC));
410        f.tenants
411            .remove(Actor::Cli, gone, crate::tenants::Removal::Empty)
412            .expect("removed");
413        // A removal from before the master key's rotation, and a refused one.
414        f.conn()
415            .execute_batch(
416                "INSERT INTO audit (at, actor, action, target, version, status, count)
417                 VALUES ('2026-10-01T00:00:00Z', 'cli', 'tenant_remove', NULL, NULL, 204, 1),
418                        ('2026-10-24T09:00:00Z', 'cli', 'tenant_remove', NULL, NULL, 409, 1);",
419            )
420            .expect("older records");
421        let found = f
422            .tenants
423            .examine(Retention::Years(7), datetime!(2026-10-25 12:00 UTC))
424            .expect("examined");
425        assert_eq!(found.removed_since_master, 1);
426        assert_eq!(
427            found.oldest_removed_since_master,
428            Some(datetime!(2026-10-24 08:00 UTC))
429        );
430        assert_eq!(found.tenants.len(), 1);
431    }
432}
vcon_server/tenants/examine.rs

vcon_server/tenants/
examine.rs
