vcon_server/tenants/
examine.rs

1//! What `check` reads of the registry: each tenant's Store examined under
2//! its own retention period, rotations left part way, and tenant removals
3//! since the master key was last rotated. Reading changes nothing.
4
5use rusqlite::OptionalExtension;
6use time::OffsetDateTime;
7use time::format_description::well_known::Rfc3339;
8
9use super::{OpenFailure, TenantError, Tenants, io, stored_retention};
10use crate::audit::Action;
11use crate::clock::from_nanos;
12use crate::envelope::{self, Context};
13use crate::ids::TenantId;
14use crate::retention::Retention;
15use crate::store::Examination;
16
17/// One tenant as `check` found it.
18#[derive(Debug)]
19pub struct TenantExamination {
20    pub id: TenantId,
21    pub name: String,
22    /// A key rotation of this tenant was stopped part way.
23    pub rotation_stopped: bool,
24    /// Its Store examined, or why its catalog did not open.
25    pub store: Result<Examination, OpenFailure>,
26}
27
28/// The registry as `check` found it.
29#[derive(Debug)]
30pub struct RegistryExamination {
31    pub tenants: Vec<TenantExamination>,
32    /// Tenants removed since the master key was last rotated: their keys
33    /// are destroyed only by the next rotation.
34    pub removed_since_master: u64,
35    /// When the oldest of those removals was made.
36    pub oldest_removed_since_master: Option<OffsetDateTime>,
37}
38
39/// Whether every tenant's data is under its current key, as `serve`
40/// requires before it starts: counts of tenants only, never which, and no
41/// path, since a path names a tenant.
42#[derive(Debug, Clone, PartialEq, Eq, Default)]
43pub struct KeyOrder {
44    /// Tenants whose key does not unwrap under the master key.
45    pub wrapped_elsewhere: u64,
46    /// Tenants with a key rotation stopped part way.
47    pub rotations_stopped: u64,
48    /// Tenants whose catalog does not open under their key, or with a
49    /// version row naming another key.
50    pub data_under_other_keys: u64,
51    /// Tenants whose Store did not open for a reason other than its key,
52    /// such as files this process may not use.
53    pub catalogs_unopened: u64,
54    /// The system's error for the first of those that has one, such as
55    /// `Permission denied (os error 13)`.
56    pub unopened_error: Option<String>,
57}
58
59impl KeyOrder {
60    pub fn is_sound(&self) -> bool {
61        *self == KeyOrder::default()
62    }
63}
64
65/// Work that only a key rotation finishes: purges since each tenant key's
66/// rotation, and tenant removals since the master key's. Read from the
67/// audit logs only.
68#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
69pub struct RotationBacklog {
70    pub tenants_with_purges: u64,
71    pub purges: u64,
72    pub oldest_purge: Option<OffsetDateTime>,
73    pub removals: u64,
74    pub oldest_removal: Option<OffsetDateTime>,
75}
76
77impl Tenants {
78    /// Counts the tenants whose data is not all under their current key.
79    pub fn key_order(&self) -> Result<KeyOrder, TenantError> {
80        let rows: Vec<([u8; 16], Vec<u8>, bool)> = self
81            .conn()
82            .prepare(
83                "SELECT id, wrapped_tenant_key, pending_wrapped_tenant_key IS NOT NULL FROM tenant",
84            )
85            .map_err(io)?
86            .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
87            .map_err(io)?
88            .collect::<Result<_, _>>()
89            .map_err(io)?;
90        let mut order = KeyOrder::default();
91        for (id, wrapped, pending) in rows {
92            let id = TenantId(id);
93            if envelope::unwrap(&self.master, &Context::Tenant { tenant: id }, &wrapped).is_err() {
94                order.wrapped_elsewhere += 1;
95                continue;
96            }
97            if pending {
98                order.rotations_stopped += 1;
99            }
100            let elsewhere = match self.store_or_why(id) {
101                Ok(store) => store.rows_under_other_keys().map_err(io)? > 0,
102                Err(OpenFailure::WrongKey) => true,
103                Err(failure) => {
104                    order.catalogs_unopened += 1;
105                    if let OpenFailure::Access { error, .. } = failure {
106                        order.unopened_error.get_or_insert(error.to_string());
107                    }
108                    false
109                }
110            };
111            if elsewhere {
112                order.data_under_other_keys += 1;
113            }
114        }
115        Ok(order)
116    }
117
118    /// The rotation backlog: purges per tenant since its key's rotation,
119    /// and removals since the master key's.
120    pub fn rotation_backlog(&self) -> Result<RotationBacklog, TenantError> {
121        let rows: Vec<([u8; 16], i64)> = self
122            .conn()
123            .prepare("SELECT id, key_since FROM tenant")
124            .map_err(io)?
125            .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))
126            .map_err(io)?
127            .collect::<Result<_, _>>()
128            .map_err(io)?;
129        let mut backlog = RotationBacklog::default();
130        for (id, key_since) in rows {
131            let key_since = from_nanos(key_since).ok_or(TenantError::Io)?;
132            let (purges, oldest) = self
133                .store(TenantId(id))?
134                .purges_since(key_since)
135                .map_err(io)?;
136            if purges > 0 {
137                backlog.tenants_with_purges += 1;
138                backlog.purges += purges;
139                backlog.oldest_purge = earliest(backlog.oldest_purge, oldest);
140            }
141        }
142        (backlog.removals, backlog.oldest_removal) = self.removals_since_master()?;
143        Ok(backlog)
144    }
145
146    /// Tenant removals since the master key's rotation, and the oldest.
147    fn removals_since_master(&self) -> Result<(u64, Option<OffsetDateTime>), TenantError> {
148        let (since, removals) = {
149            let conn = self.conn();
150            let since: Option<i64> = conn
151                .query_row(
152                    "SELECT since FROM master_key_use WHERE master_key_id = ?1",
153                    [self.master.id().bytes()],
154                    |r| r.get(0),
155                )
156                .optional()
157                .map_err(io)?;
158            let removals: Vec<String> = conn
159                .prepare("SELECT at FROM audit WHERE action = ?1 AND status = 204")
160                .map_err(io)?
161                .query_map([Action::TenantRemove.as_str()], |r| r.get(0))
162                .map_err(io)?
163                .collect::<Result<_, _>>()
164                .map_err(io)?;
165            (since, removals)
166        };
167        let since = since
168            .map(|s| from_nanos(s).ok_or(TenantError::Io))
169            .transpose()?;
170        let (mut count, mut oldest) = (0, None);
171        for at in removals {
172            let at = OffsetDateTime::parse(&at, &Rfc3339).map_err(io)?;
173            if since.is_none_or(|since| at >= since) {
174                count += 1;
175                oldest = earliest(oldest, Some(at));
176            }
177        }
178        Ok((count, oldest))
179    }
180
181    /// Examines every tenant at `now`, under its own retention period or
182    /// `default`.
183    pub fn examine(
184        &self,
185        default: Retention,
186        now: OffsetDateTime,
187    ) -> Result<RegistryExamination, TenantError> {
188        type Row = ([u8; 16], String, Option<String>, i64, bool);
189        let rows: Vec<Row> = self
190            .conn()
191            .prepare(
192                "SELECT id, name, retention, key_since, pending_wrapped_tenant_key IS NOT NULL
193                 FROM tenant ORDER BY name",
194            )
195            .map_err(io)?
196            .query_map([], |r| {
197                Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?))
198            })
199            .map_err(io)?
200            .collect::<Result<_, _>>()
201            .map_err(io)?;
202        let time = |nanos| from_nanos(nanos).ok_or(TenantError::Io);
203        let mut tenants = Vec::new();
204        for (id, name, retention, key_since, rotation_stopped) in rows {
205            let id = TenantId(id);
206            let retention = stored_retention(retention)?.unwrap_or(default);
207            let key_since = time(key_since)?;
208            let store = self.store_or_why(id).and_then(|store| {
209                store
210                    .examine(retention, key_since, now)
211                    .map_err(|e| OpenFailure::Refused(io(e)))
212            });
213            tenants.push(TenantExamination {
214                id,
215                name,
216                rotation_stopped,
217                store,
218            });
219        }
220        let (removed_since_master, oldest_removed_since_master) = self.removals_since_master()?;
221        let found = RegistryExamination {
222            tenants,
223            removed_since_master,
224            oldest_removed_since_master,
225        };
226        Ok(found)
227    }
228}
229
230/// The earlier of two times, either of which may be absent.
231fn earliest(a: Option<OffsetDateTime>, b: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
232    match (a, b) {
233        (Some(a), Some(b)) => Some(a.min(b)),
234        (a, b) => a.or(b),
235    }
236}
237
238#[cfg(test)]
239mod examine_tests {
240    use time::macros::datetime;
241
242    use super::super::tests::fixture;
243    use super::*;
244    use crate::audit::Actor;
245    use crate::store::Caller;
246    use crate::store::testing::vcon;
247
248    const CALLER: Caller = Caller {
249        key: crate::ids::KeyId([7; 8]),
250    };
251
252    #[test]
253    fn each_tenant_is_examined_under_its_own_period_or_the_default() {
254        let f = fixture();
255        let own = f
256            .tenants
257            .create(Actor::Cli, "own", Some(Retention::Days(1)))
258            .expect("made");
259        let default = f.tenants.create(Actor::Cli, "default", None).expect("made");
260        for id in [own, default] {
261            let store = f.tenants.store(id).expect("store");
262            let (found, body) = vcon("01a07da8-c2bb-83e5-b9a2-279e0d16bc46", "kept");
263            store.create(&CALLER, &found, &body).expect("stored");
264            store
265                .delete(&CALLER, found.uuid, Retention::Years(7))
266                .expect("deleted");
267        }
268        let found = f
269            .tenants
270            .examine(Retention::Years(7), datetime!(2026-10-30 12:00 UTC))
271            .expect("examined");
272        let by_name = |name: &str| found.tenants.iter().find(|t| t.name == name).expect(name);
273        let own = by_name("own");
274        assert_eq!(
275            own.store.as_ref().expect("opens").due,
276            1,
277            "a day has passed"
278        );
279        assert!(!own.rotation_stopped);
280        assert_eq!(by_name("default").store.as_ref().expect("opens").due, 0);
281        assert_eq!(found.tenants.len(), 2);
282    }
283
284    #[test]
285    fn a_rotation_stopped_part_way_is_reported_and_a_catalog_that_does_not_open_too() {
286        let f = fixture();
287        let id = f.tenants.create(Actor::Cli, "acme", None).expect("made");
288        *f.tenants.hooks.rekey_stop.lock().expect("hook") =
289            Some(super::super::rekey::RekeyStep::Reencrypted);
290        assert_eq!(f.tenants.rekey_tenant(Actor::Cli, id), Err(TenantError::Io));
291        let found = f
292            .tenants
293            .examine(Retention::Years(7), datetime!(2026-10-23 12:00 UTC))
294            .expect("examined");
295        let acme = &found.tenants[0];
296        assert!(acme.rotation_stopped);
297        assert!(
298            matches!(acme.store, Err(OpenFailure::WrongKey)),
299            "its catalog is under the pending key: {:?}",
300            acme.store
301        );
302    }
303
304    /// A tenant's files this process may not write, such as ones a command
305    /// run as root left, are reported as that, with the path and the
306    /// system's error; only a key that does not open the catalog is a key
307    /// problem.
308    #[test]
309    fn a_catalog_the_process_may_not_open_is_an_access_failure_not_a_key_one() {
310        use std::os::unix::fs::PermissionsExt;
311        let f = fixture();
312        // Made, but not yet opened through the registry, as at start-up.
313        let id = f.tenants.create(Actor::Cli, "acme", None).expect("made");
314        let dir = f.dir.path().join("tenants").join(id.to_string());
315        let mode =
316            |m| std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(m)).expect("chmod");
317        mode(0o500);
318        let order = f.tenants.key_order();
319        let found = f
320            .tenants
321            .examine(Retention::Years(7), datetime!(2026-10-23 12:00 UTC));
322        mode(0o700);
323        let order = order.expect("read");
324        assert_eq!(order.data_under_other_keys, 0, "{order:?}");
325        assert_eq!(order.catalogs_unopened, 1, "{order:?}");
326        assert_eq!(
327            order.unopened_error.as_deref(),
328            Some("Permission denied (os error 13)")
329        );
330        assert!(!order.is_sound());
331        match &found.expect("examined").tenants[0].store {
332            Err(OpenFailure::Access { path, error }) => {
333                assert_eq!(path, &dir);
334                assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied);
335            }
336            other => panic!("{other:?}"),
337        }
338        assert!(
339            f.tenants.key_order().expect("read").is_sound(),
340            "once allowed"
341        );
342    }
343
344    mod key_order {
345        use super::*;
346        use crate::envelope::{self, Context, Key};
347        use crate::tenants::rekey::RekeyStep;
348
349        fn with_vcons(f: &super::super::super::tests::Fixture) -> TenantId {
350            let id = f.tenants.create(Actor::Cli, "acme", None).expect("made");
351            let store = f.tenants.store(id).expect("store");
352            for n in 0..3 {
353                let (found, body) = vcon(&format!("01a07da8-c2bb-83e5-b9a2-279e0d16bc4{n}"), "k");
354                store.create(&CALLER, &found, &body).expect("stored");
355            }
356            id
357        }
358
359        #[test]
360        fn a_sound_registry_is_in_order() {
361            let f = fixture();
362            with_vcons(&f);
363            assert!(f.tenants.key_order().expect("read").is_sound());
364        }
365
366        #[test]
367        fn a_rotation_stopped_at_any_step_is_out_of_order() {
368            for (step, data) in [
369                (RekeyStep::Pending, 0),
370                (RekeyStep::Batch, 1),
371                (RekeyStep::Rewrapped, 1),
372                (RekeyStep::Reencrypted, 1),
373            ] {
374                let f = fixture();
375                let id = with_vcons(&f);
376                *f.tenants.hooks.rekey_stop.lock().expect("hook") = Some(step);
377                assert!(f.tenants.rekey_in_batches(Actor::Cli, id, 2).is_err());
378                assert_eq!(
379                    f.tenants.key_order().expect("read"),
380                    KeyOrder {
381                        wrapped_elsewhere: 0,
382                        rotations_stopped: 1,
383                        data_under_other_keys: data,
384                        ..KeyOrder::default()
385                    },
386                    "{step:?}"
387                );
388                f.tenants.rekey_tenant(Actor::Cli, id).expect("finished");
389                assert!(f.tenants.key_order().expect("read").is_sound(), "{step:?}");
390            }
391        }
392
393        #[test]
394        fn a_tenant_key_wrapped_under_another_key_is_out_of_order() {
395            let f = fixture();
396            let id = with_vcons(&f);
397            let other = Key::generate().expect("key");
398            let foreign =
399                envelope::wrap(&other, &Context::Tenant { tenant: id }, &f.tenant_key(id));
400            f.conn()
401                .execute(
402                    "UPDATE tenant SET wrapped_tenant_key = ?1 WHERE id = ?2",
403                    rusqlite::params![foreign, id.0],
404                )
405                .expect("rewritten");
406            f.tenants.close(id);
407            assert_eq!(f.tenants.key_order().expect("read").wrapped_elsewhere, 1);
408        }
409    }
410
411    #[test]
412    fn the_rotation_backlog_counts_purges_and_removals_from_the_logs() {
413        let f = fixture();
414        let purging = f
415            .tenants
416            .create(Actor::Cli, "purging", Some(Retention::Days(0)))
417            .expect("made");
418        f.tenants.create(Actor::Cli, "quiet", None).expect("made");
419        let gone = f.tenants.create(Actor::Cli, "gone", None).expect("made");
420        let store = f.tenants.store(purging).expect("store");
421        for (n, at) in [
422            (0, datetime!(2026-10-24 10:00 UTC)),
423            (1, datetime!(2026-10-24 11:00 UTC)),
424        ] {
425            f.clock.set(at);
426            let (found, body) = vcon(&format!("01a07da8-c2bb-83e5-b9a2-279e0d16bc4{n}"), "k");
427            store.create(&CALLER, &found, &body).expect("stored");
428            store
429                .delete(&CALLER, found.uuid, Retention::Days(0))
430                .expect("deleted and purged");
431        }
432        drop(store);
433        f.clock.set(datetime!(2026-10-24 12:00 UTC));
434        f.tenants
435            .remove(Actor::Cli, gone, crate::tenants::Removal::Empty)
436            .expect("removed");
437        assert_eq!(
438            f.tenants.rotation_backlog().expect("read"),
439            RotationBacklog {
440                tenants_with_purges: 1,
441                purges: 2,
442                oldest_purge: Some(datetime!(2026-10-24 10:00 UTC)),
443                removals: 1,
444                oldest_removal: Some(datetime!(2026-10-24 12:00 UTC)),
445            }
446        );
447    }
448
449    #[test]
450    fn removals_since_the_master_key_was_rotated_are_counted() {
451        let f = fixture();
452        let gone = f.tenants.create(Actor::Cli, "gone", None).expect("made");
453        f.tenants.create(Actor::Cli, "kept", None).expect("made");
454        f.clock.set(datetime!(2026-10-24 08:00 UTC));
455        f.tenants
456            .remove(Actor::Cli, gone, crate::tenants::Removal::Empty)
457            .expect("removed");
458        // A removal from before the master key's rotation, and a refused one.
459        f.conn()
460            .execute_batch(
461                "INSERT INTO audit (at, actor, action, target, version, status, count)
462                 VALUES ('2026-10-01T00:00:00Z', 'cli', 'tenant_remove', NULL, NULL, 204, 1),
463                        ('2026-10-24T09:00:00Z', 'cli', 'tenant_remove', NULL, NULL, 409, 1);",
464            )
465            .expect("older records");
466        let found = f
467            .tenants
468            .examine(Retention::Years(7), datetime!(2026-10-25 12:00 UTC))
469            .expect("examined");
470        assert_eq!(found.removed_since_master, 1);
471        assert_eq!(
472            found.oldest_removed_since_master,
473            Some(datetime!(2026-10-24 08:00 UTC))
474        );
475        assert_eq!(found.tenants.len(), 1);
476    }
477}
vcon_server/tenants/examine.rs

vcon_server/tenants/
examine.rs
