vcon_server/tenants/
rekey.rs

1//! A tenant key's rotation: a new tenant key replaces the old one, every
2//! data key is rewrapped under it, and the tenant's catalog is re-encrypted
3//! under its catalog key, so the old key opens nothing that is kept. A run
4//! stopped at any step is finished by running it again.
5
6use std::path::Path;
7
8use time::OffsetDateTime;
9
10use super::{TenantError, Tenants, commit, io};
11use crate::audit::{Action, Actor, Entry};
12use crate::backend::Under;
13use crate::clock::nanos;
14use crate::envelope::{self, Context, Key, WrapLimit};
15use crate::ids::TenantId;
16use crate::store::StoreError;
17
18/// How many data keys one catalog transaction rewraps.
19pub(super) const REKEY_BATCH: usize = 1000;
20
21/// The steps a rotation may be stopped after, for tests.
22#[derive(Debug, Clone, Copy, PartialEq, Eq)]
23pub(crate) enum RekeyStep {
24    /// The new key is committed as pending.
25    Pending,
26    /// One batch of data keys is rewrapped.
27    Batch,
28    /// Every data key is rewrapped.
29    Rewrapped,
30    /// The catalog is re-encrypted.
31    Reencrypted,
32}
33
34/// The steps a master rotation may be stopped after, for tests.
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
36pub(crate) enum MasterStep {
37    /// server.sqlite is copied under the new master key's catalog key.
38    Copied,
39    /// Every tenant key in the copy is rewrapped.
40    Rewrapped,
41    /// The copy has replaced server.sqlite.
42    Renamed,
43}
44
45/// Rotates the master key of the data directory `dir`: server.sqlite is
46/// copied under `new`'s catalog key, every tenant key in the copy, pending
47/// ones too, is rewrapped from `old` to `new`, and the copy is synced and
48/// renamed over server.sqlite, so until the rename the old file is
49/// untouched. A run stopped before the rename starts again; one stopped
50/// after it finds server.sqlite under `new` and has nothing left to do.
51/// Only the command line does this, with the server stopped.
52pub fn rekey_master(
53    dir: &Path,
54    old: &Key,
55    new: &Key,
56    now: OffsetDateTime,
57) -> Result<(), TenantError> {
58    rekey_master_stopping(dir, old, new, now, None)
59}
60
61pub(crate) fn rekey_master_stopping(
62    dir: &Path,
63    old: &Key,
64    new: &Key,
65    now: OffsetDateTime,
66    stop: Option<MasterStep>,
67) -> Result<(), TenantError> {
68    crate::backend::local::rekey_master(dir, old, new, now, stop)
69}
70
71impl Tenants {
72    /// Rotates the tenant's key. Only the command line does this, with the
73    /// server stopped.
74    pub fn rekey_tenant(&self, who: Actor, id: TenantId) -> Result<(), TenantError> {
75        self.rekey_in_batches(who, id, REKEY_BATCH)
76    }
77
78    pub(super) fn rekey_in_batches(
79        &self,
80        who: Actor,
81        id: TenantId,
82        batch: usize,
83    ) -> Result<(), TenantError> {
84        let entry = |status| Entry {
85            actor: who,
86            action: Action::RekeyTenant,
87            target: Some(id.to_string()),
88            version: None,
89            status,
90        };
91        let ctx = Context::Tenant { tenant: id };
92
93        // 1. The new key, committed as pending before anything uses it, so a
94        // stopped run's data keys are never wrapped under a key that is lost.
95        let (old, new) = {
96            let mut tx = self.registry.immediate()?;
97            let Some(row) = tx.tenant(id)? else {
98                self.record(&mut tx, &entry(404))?;
99                commit(tx)?;
100                return Err(TenantError::NotFound);
101            };
102            let (wrapped, pending) = (row.wrapped, row.pending);
103            let old = envelope::unwrap(&self.master, &ctx, &wrapped).map_err(io)?;
104            let new = match pending {
105                Some(pending) => envelope::unwrap(&self.master, &ctx, &pending).map_err(io)?,
106                None => {
107                    let master_id = self.master.id();
108                    if envelope::wrap_limit(tx.master_wraps(master_id)?) == WrapLimit::Refuse {
109                        self.record(&mut tx, &entry(503))?;
110                        commit(tx)?;
111                        return Err(TenantError::LimitReached);
112                    }
113                    tx.add_master_wrap(master_id, nanos(self.clock.now()))?;
114                    let new = Key::generate().map_err(io)?;
115                    let pending = envelope::wrap(&self.master, &ctx, &new);
116                    tx.set_pending(id, &pending)?;
117                    new
118                }
119            };
120            commit(tx)?;
121            (old, new)
122        };
123        self.stop_after(RekeyStep::Pending)?;
124
125        // No Store handle may outlive the old key; the next use of the
126        // tenant opens it afresh under whichever key is current.
127        drop(self.drain(id)?);
128        let moved = self.move_catalog(id, old, &new, batch);
129        self.stores().remove(&id);
130        moved?;
131
132        // 4. The new key becomes current and the old one goes, in one
133        // transaction with its record.
134        {
135            let mut tx = self.registry.immediate()?;
136            tx.promote_pending(id, new.id(), nanos(self.clock.now()))?;
137            self.record(&mut tx, &entry(200))?;
138            commit(tx)?;
139        }
140        self.copy_to_tenant(id, &entry(200))
141    }
142
143    /// 2 and 3: every data key rewrapped from `old` to `new`, then the
144    /// catalog re-encrypted under `new`'s catalog key. A catalog already under it
145    /// was finished by a run that stopped after; it is left as it is.
146    fn move_catalog(
147        &self,
148        id: TenantId,
149        old: Key,
150        new: &Key,
151        batch: usize,
152    ) -> Result<(), TenantError> {
153        let old_catalog = envelope::catalog_key(&old).map_err(io)?;
154        let new_catalog = envelope::catalog_key(new).map_err(io)?;
155        if self.space.catalog_under(id, &old_catalog, &new_catalog)? == Under::New {
156            return Ok(());
157        }
158        let store = self
159            .space
160            .open(id, old, true, self.clock.clone())
161            .map_err(io)?;
162        loop {
163            let done = store.rewrap(new, batch).map_err(|e| match e {
164                StoreError::LimitReached => TenantError::LimitReached,
165                other => io(other),
166            })?;
167            if done == 0 {
168                break;
169            }
170            self.stop_after(RekeyStep::Batch)?;
171        }
172        drop(store);
173        self.stop_after(RekeyStep::Rewrapped)?;
174        self.space.reseal(id, &old_catalog, &new_catalog)?;
175        self.stop_after(RekeyStep::Reencrypted)
176    }
177
178    /// In tests, fails the rotation after `step` when a test asked for it.
179    fn stop_after(&self, step: RekeyStep) -> Result<(), TenantError> {
180        #[cfg(test)]
181        {
182            let mut stop = self.hooks.rekey_stop.lock().expect("hook");
183            if *stop == Some(step) {
184                *stop = None;
185                return Err(TenantError::Io);
186            }
187        }
188        let _ = step;
189        Ok(())
190    }
191}
192
193#[cfg(test)]
194mod rekey_tests {
195    use uuid::Uuid;
196
197    use super::super::tests::{Fixture, fixture};
198    use super::*;
199    use crate::catalog::{self, CatalogError};
200    use crate::envelope::{self, Context, Key};
201    use crate::ids::VersionId;
202    use crate::store::testing::vcon;
203    use crate::store::{Caller, Created};
204
205    /// A version row as stored: version, uuid, wrapping key id, wrapped key.
206    type RawVersion = (Vec<u8>, Vec<u8>, Vec<u8>, Vec<u8>);
207
208    const CALLER: Caller = Caller {
209        key: crate::ids::KeyId([7; 8]),
210    };
211
212    /// A tenant with five vCons, one of them deleted, so a hidden version
213    /// is kept too.
214    fn tenant_with_vcons(f: &Fixture) -> (TenantId, Vec<Uuid>) {
215        let id = f.tenants.create(Actor::Cli, "acme", None).expect("made");
216        let store = f.tenants.store(id).expect("store");
217        let mut uuids = Vec::new();
218        for n in 0..5 {
219            let (found, body) = vcon(&format!("01a07da8-c2bb-83e5-b9a2-279e0d16bc4{n}"), "kept");
220            assert!(matches!(
221                store.create(&CALLER, &found, &body),
222                Ok(Created::New(_))
223            ));
224            uuids.push(found.uuid);
225        }
226        store
227            .delete(&CALLER, uuids[4], crate::retention::Retention::Years(7))
228            .expect("deleted");
229        (id, uuids)
230    }
231
232    /// Every row of the tenant's catalog opens under the current key, and
233    /// nothing kept opens under `old`.
234    fn all_under_the_new_key_only(f: &Fixture, id: TenantId, uuids: &[Uuid], old: &Key) {
235        let store = f.tenants.store(id).expect("store");
236        for uuid in &uuids[..4] {
237            assert!(store.get(&CALLER, *uuid).is_ok(), "{uuid} reads");
238        }
239        assert!(
240            store.legal_get(&CALLER, uuids[4], None).is_ok(),
241            "the deleted vCon's kept version reads"
242        );
243        let path = f.tenants_dir().join(id.to_string()).join("catalog.sqlite");
244        let old_catalog = envelope::catalog_key(old).expect("catalog key");
245        assert!(matches!(
246            catalog::open(&path, &old_catalog, ""),
247            Err(CatalogError::WrongKey)
248        ));
249        let new = f.tenant_key(id);
250        assert_ne!(new.id(), old.id());
251        let conn = catalog::open(&path, &envelope::catalog_key(&new).expect("key"), "")
252            .expect("opens under the new key");
253        let rows: Vec<RawVersion> = conn
254            .prepare("SELECT version, uuid, tenant_key_id, wrapped_key FROM version")
255            .expect("prepare")
256            .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))
257            .expect("query")
258            .collect::<Result<_, _>>()
259            .expect("rows");
260        assert_eq!(rows.len(), 5);
261        for (version, uuid, key_id, wrapped) in rows {
262            let ctx = Context::Version {
263                tenant: id,
264                uuid: Uuid::from_slice(&uuid).expect("uuid"),
265                version: VersionId(version.try_into().expect("16 bytes")),
266            };
267            assert_eq!(key_id, new.id().bytes().to_vec());
268            assert!(envelope::unwrap(&new, &ctx, &wrapped).is_ok());
269            assert!(envelope::unwrap(old, &ctx, &wrapped).is_err());
270        }
271        let pending: Option<Vec<u8>> = f
272            .conn()
273            .query_row(
274                "SELECT pending_wrapped_tenant_key FROM tenant WHERE id = ?1",
275                [id.0],
276                |r| r.get(0),
277            )
278            .expect("row");
279        assert_eq!(pending, None, "no key is left pending");
280    }
281
282    #[test]
283    fn a_rekey_moves_everything_kept_to_a_new_key_and_records_it() {
284        let f = fixture();
285        let (id, uuids) = tenant_with_vcons(&f);
286        let old = f.tenant_key(id);
287        let since_before: i64 = f
288            .conn()
289            .query_row("SELECT key_since FROM tenant WHERE id = ?1", [id.0], |r| {
290                r.get(0)
291            })
292            .expect("row");
293        f.clock.set(time::macros::datetime!(2026-10-23 13:00 UTC));
294        f.tenants.rekey_tenant(Actor::Cli, id).expect("rekeyed");
295        all_under_the_new_key_only(&f, id, &uuids, &old);
296        let since_after: i64 = f
297            .conn()
298            .query_row("SELECT key_since FROM tenant WHERE id = ?1", [id.0], |r| {
299                r.get(0)
300            })
301            .expect("row");
302        assert!(
303            since_after > since_before,
304            "key_since moves to the rotation"
305        );
306        assert_eq!(
307            since_after,
308            crate::clock::nanos(time::macros::datetime!(2026-10-23 13:00 UTC)),
309            "key_since is the rotation's own time"
310        );
311        let records = f.admin_records("rekey_tenant");
312        assert_eq!(records.len(), 1);
313        assert_eq!(records[0]["status"], 200);
314        assert_eq!(records[0]["target"], id.to_string().as_str());
315        let own = crate::audit::testing::exported(&f.store(id).conn_for_test(), None);
316        let copies: Vec<_> = own
317            .iter()
318            .filter(|r| r["action"] == "rekey_tenant")
319            .collect();
320        assert_eq!(copies.len(), 1, "the tenant's log keeps one copy");
321        assert_eq!(copies[0]["status"], 200, "the copy records the success");
322        assert_eq!(
323            copies[0]["target"],
324            id.to_string().as_str(),
325            "the copy names the tenant"
326        );
327        assert_eq!(f.master_wraps(), 2, "the new tenant key's wrap is counted");
328    }
329
330    #[test]
331    fn a_rekey_stopped_at_any_step_is_finished_by_running_it_again() {
332        for step in [
333            RekeyStep::Pending,
334            RekeyStep::Batch,
335            RekeyStep::Rewrapped,
336            RekeyStep::Reencrypted,
337        ] {
338            let f = fixture();
339            let (id, uuids) = tenant_with_vcons(&f);
340            let old = f.tenant_key(id);
341            *f.tenants.hooks.rekey_stop.lock().expect("hook") = Some(step);
342            assert_eq!(
343                f.tenants.rekey_in_batches(Actor::Cli, id, 2),
344                Err(TenantError::Io),
345                "{step:?}"
346            );
347            f.tenants
348                .rekey_in_batches(Actor::Cli, id, 2)
349                .unwrap_or_else(|e| panic!("{step:?}: {e:?}"));
350            all_under_the_new_key_only(&f, id, &uuids, &old);
351            assert_eq!(f.master_wraps(), 2, "{step:?}: one new key, made once");
352        }
353    }
354
355    mod master {
356        use std::path::{Path, PathBuf};
357        use std::sync::Arc;
358
359        use super::*;
360        use crate::clock::ManualClock;
361        use crate::fs::OsFs;
362
363        const NOW: OffsetDateTime = time::macros::datetime!(2026-10-25 12:00 UTC);
364
365        struct Dir {
366            dir: tempfile::TempDir,
367        }
368
369        impl Dir {
370            fn data(&self) -> &Path {
371                self.dir.path()
372            }
373            fn key_file(&self, name: &str) -> PathBuf {
374                self.dir.path().join(name)
375            }
376            /// A master key, loaded afresh from its file each time.
377            fn key(&self, name: &str) -> Key {
378                Key::load_master(&self.key_file(name)).expect("a key file")
379            }
380            fn open(&self, name: &str) -> Result<Tenants, crate::tenants::RegistryError> {
381                Tenants::open(
382                    self.data(),
383                    self.key(name),
384                    Arc::new(OsFs),
385                    Arc::new(ManualClock::at(NOW)),
386                )
387            }
388            fn server_file(&self) -> Vec<u8> {
389                std::fs::read(self.data().join("server.sqlite")).expect("server.sqlite")
390            }
391        }
392
393        /// Two tenants with a vCon each, one of them with a tenant rotation
394        /// stopped after its catalog was re-encrypted under the pending
395        /// key, under master key "old". That tenant is readable only through
396        /// its pending key, so a master rotation must carry it.
397        fn setup() -> (Dir, Vec<(TenantId, Uuid)>, TenantId) {
398            let d = Dir {
399                dir: tempfile::tempdir().expect("temp dir"),
400            };
401            std::fs::write(d.key_file("old"), [0x31; 32]).expect("old key");
402            std::fs::write(d.key_file("new"), [0x32; 32]).expect("new key");
403            let tenants = d.open("old").expect("opens");
404            let mut made = Vec::new();
405            for (n, name) in ["acme", "beta"].into_iter().enumerate() {
406                let id = tenants.create(Actor::Cli, name, None).expect("made");
407                let (found, body) = vcon(&format!("01a07da8-c2bb-83e5-b9a2-279e0d16bc4{n}"), name);
408                tenants
409                    .store(id)
410                    .expect("store")
411                    .create(&CALLER, &found, &body)
412                    .expect("stored");
413                made.push((id, found.uuid));
414            }
415            let pending = made[1].0;
416            *tenants.hooks.rekey_stop.lock().expect("hook") = Some(RekeyStep::Reencrypted);
417            assert_eq!(
418                tenants.rekey_tenant(Actor::Cli, pending),
419                Err(TenantError::Io)
420            );
421            drop(tenants);
422            (d, made, pending)
423        }
424
425        /// Everything opens under "new", and "old" opens nothing.
426        fn rotated(d: &Dir, made: &[(TenantId, Uuid)], pending: TenantId) {
427            assert!(
428                matches!(d.open("old"), Err(crate::tenants::RegistryError::WrongKey)),
429                "after the rotation, server.sqlite is under the new master key only"
430            );
431            let tenants = d.open("new").expect("opens under the new master key");
432            // The stopped tenant rotation finishes under the new master key.
433            tenants
434                .rekey_tenant(Actor::Cli, pending)
435                .expect("the pending tenant key unwraps under the new master");
436            for (id, uuid) in made {
437                let store = tenants.store(*id).expect("store");
438                assert!(store.get(&CALLER, *uuid).is_ok(), "{id} reads");
439            }
440            let records = tenants.admin_records();
441            let rotations: Vec<_> = records
442                .iter()
443                .filter(|r| r["action"] == "rekey_master")
444                .collect();
445            assert_eq!(rotations.len(), 1);
446            assert_eq!(rotations[0]["status"], 200);
447            assert_eq!(rotations[0]["actor"], "cli");
448            let uses: Vec<(Vec<u8>, i64)> = tenants
449                .conn_for_test()
450                .prepare("SELECT master_key_id, wraps FROM master_key_use")
451                .expect("prepare")
452                .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))
453                .expect("query")
454                .collect::<Result<_, _>>()
455                .expect("rows");
456            assert_eq!(
457                uses,
458                [(d.key("new").id().bytes().to_vec(), 3)],
459                "two tenant keys and one pending one, wrapped under the new key only"
460            );
461        }
462
463        #[test]
464        fn a_master_rotation_moves_every_tenant_key_to_the_new_master() {
465            let (d, made, pending) = setup();
466            rekey_master(d.data(), &d.key("old"), &d.key("new"), NOW).expect("rotated");
467            rotated(&d, &made, pending);
468        }
469
470        #[test]
471        fn stopped_before_the_rename_the_old_file_is_untouched_and_a_rerun_finishes() {
472            for step in [MasterStep::Copied, MasterStep::Rewrapped] {
473                let (d, made, pending) = setup();
474                let before = d.server_file();
475                assert_eq!(
476                    rekey_master_stopping(d.data(), &d.key("old"), &d.key("new"), NOW, Some(step)),
477                    Err(TenantError::Io),
478                    "{step:?}"
479                );
480                assert_eq!(
481                    d.server_file(),
482                    before,
483                    "{step:?}: the old file is untouched"
484                );
485                assert!(d.open("old").is_ok(), "{step:?}: still under the old key");
486                rekey_master(d.data(), &d.key("old"), &d.key("new"), NOW).expect("rotated");
487                rotated(&d, &made, pending);
488            }
489        }
490
491        #[test]
492        fn stopped_after_the_rename_a_rerun_has_nothing_left_to_do() {
493            let (d, made, pending) = setup();
494            assert_eq!(
495                rekey_master_stopping(
496                    d.data(),
497                    &d.key("old"),
498                    &d.key("new"),
499                    NOW,
500                    Some(MasterStep::Renamed)
501                ),
502                Err(TenantError::Io)
503            );
504            let after = d.server_file();
505            rekey_master(d.data(), &d.key("old"), &d.key("new"), NOW).expect("done already");
506            assert_eq!(d.server_file(), after, "nothing is rewritten");
507            rotated(&d, &made, pending);
508        }
509
510        #[test]
511        fn a_key_that_opens_neither_way_is_refused() {
512            let (d, _, _) = setup();
513            std::fs::write(d.key_file("other"), [0x33; 32]).expect("other key");
514            let before = d.server_file();
515            assert_eq!(
516                rekey_master(d.data(), &d.key("other"), &d.key("new"), NOW),
517                Err(TenantError::WrongKey)
518            );
519            assert_eq!(d.server_file(), before);
520        }
521    }
522
523    #[test]
524    fn an_unknown_tenant_is_refused_and_recorded() {
525        let f = fixture();
526        let unknown = TenantId::generate();
527        assert_eq!(
528            f.tenants.rekey_tenant(Actor::Cli, unknown),
529            Err(TenantError::NotFound)
530        );
531        assert_eq!(f.admin_records("rekey_tenant")[0]["status"], 404);
532    }
533
534    #[test]
535    fn a_master_key_at_its_wrap_limit_makes_no_new_tenant_key() {
536        let f = fixture();
537        let (id, _) = tenant_with_vcons(&f);
538        f.conn()
539            .execute("UPDATE master_key_use SET wraps = ?1", [1_i64 << 32])
540            .expect("at the limit");
541        assert_eq!(
542            f.tenants.rekey_tenant(Actor::Cli, id),
543            Err(TenantError::LimitReached)
544        );
545        assert_eq!(f.admin_records("rekey_tenant")[0]["status"], 503);
546        let pending: Option<Vec<u8>> = f
547            .conn()
548            .query_row(
549                "SELECT pending_wrapped_tenant_key FROM tenant WHERE id = ?1",
550                [id.0],
551                |r| r.get(0),
552            )
553            .expect("row");
554        assert_eq!(pending, None);
555        assert!(
556            f.tenants.store(id).is_ok(),
557            "the tenant is served as before"
558        );
559    }
560}
vcon_server/tenants/rekey.rs

vcon_server/tenants/
rekey.rs
