vcon_server/tenants/
backup.rs

1//! A copy of the whole data directory, made while a server may be running:
2//! server.sqlite with SQLite's online backup API, then each tenant it lists,
3//! as `Store::backup_to` copies one.
4
5use std::collections::BTreeSet;
6use std::io;
7use std::path::Path;
8
9use super::{TenantError, Tenants};
10use crate::audit::{Action, Actor, Entry};
11use crate::catalog;
12use crate::envelope;
13use crate::ids::TenantId;
14use crate::store::{self, TRIES};
15
16/// What `backup` copied.
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub struct Backup {
19    pub tenants: u64,
20    pub versions: u64,
21    /// How many times server.sqlite was copied.
22    pub tries: usize,
23}
24
25/// Why a backup was refused or failed.
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub enum BackupError {
28    /// Something is already at the backup's path.
29    Exists,
30    /// Tenants were removed, or versions purged, after each try's copy.
31    KeptChanging,
32    /// A file could not be read, written or synced, or a catalog could not
33    /// be copied or read.
34    Io,
35    /// The admin log could not record the backup.
36    AuditFailed,
37}
38
39impl Tenants {
40    /// Copies the data directory into `to`, which must not exist, and
41    /// records the run in the admin log with its outcome.
42    ///
43    /// Each try copies server.sqlite in one step inside one read
44    /// transaction, then each tenant that copy lists, as `Store::backup_to`
45    /// copies one. A tenant the copy lists that is gone, or whose versions
46    /// kept being purged, was removed meanwhile, and the next try copies
47    /// server.sqlite again. Tenant directories the final copy does not
48    /// list are removed. Tenants are copied one after another, each as it
49    /// was at its own copy.
50    pub fn backup(&self, who: Actor, to: &Path) -> Result<Backup, BackupError> {
51        let done = self.copy_all(to);
52        let status = match done {
53            Ok(_) => 200,
54            Err(BackupError::Exists) => 409,
55            Err(BackupError::KeptChanging) => 503,
56            Err(BackupError::Io | BackupError::AuditFailed) => 500,
57        };
58        self.record_alone(&Entry {
59            actor: who,
60            action: Action::Backup,
61            target: None,
62            version: None,
63            status,
64        })
65        .map_err(|_| BackupError::AuditFailed)?;
66        done
67    }
68
69    fn copy_all(&self, to: &Path) -> Result<Backup, BackupError> {
70        let fail = |_| BackupError::Io;
71        match self.fs.list(to) {
72            Err(e) if e.kind() == io::ErrorKind::NotFound => {}
73            Ok(_) => return Err(BackupError::Exists),
74            Err(e) if e.kind() == io::ErrorKind::NotADirectory => return Err(BackupError::Exists),
75            Err(_) => return Err(BackupError::Io),
76        }
77        let key = envelope::catalog_key(&self.master).map_err(|_| BackupError::Io)?;
78        let tenants = to.join("tenants");
79        self.fs.create_dir_all(&tenants).map_err(fail)?;
80        // Only the service account may enter the copy, as with the data
81        // directory, which is restored by copying this one with its modes.
82        {
83            use std::os::unix::fs::PermissionsExt;
84            std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o700)).map_err(fail)?;
85        }
86        let registry = to.join("server.sqlite");
87        'tries: for tries in 1..=TRIES {
88            for file in ["server.sqlite", "server.sqlite-wal", "server.sqlite-shm"] {
89                remove_if_there(&to.join(file)).map_err(fail)?;
90            }
91            catalog::backup(&self.conn(), &registry, &key).map_err(|_| BackupError::Io)?;
92            #[cfg(test)]
93            if let Some(hook) = self.hooks.registry_copied.lock().expect("hook").as_mut() {
94                hook(self);
95            }
96            let listed = tenants_in(&registry, &key)?;
97            let mut versions = 0;
98            for id in &listed {
99                let copied = match self.store(*id) {
100                    Ok(store) => store.backup_to(&tenants.join(id.to_string())),
101                    Err(TenantError::NotFound | TenantError::Closing) => continue 'tries,
102                    Err(_) => return Err(BackupError::Io),
103                };
104                match copied {
105                    Ok(copied) => versions += copied.versions,
106                    // A tenant removed while it was copied: its files went
107                    // with it, so no try of its own could finish.
108                    Err(store::BackupError::KeptChanging) if !self.listed(*id)? => {
109                        self.stores().remove(id);
110                        continue 'tries;
111                    }
112                    Err(store::BackupError::KeptChanging) => return Err(BackupError::KeptChanging),
113                    Err(store::BackupError::Io) => return Err(BackupError::Io),
114                }
115            }
116            let names: BTreeSet<String> = listed.iter().map(ToString::to_string).collect();
117            for name in self.fs.list(&tenants).map_err(fail)? {
118                let name = name.into_string().unwrap_or_default();
119                if !names.contains(&name) {
120                    self.fs.remove_dir_all(&tenants.join(&name)).map_err(fail)?;
121                }
122            }
123            self.fs.sync_dir(&tenants).map_err(fail)?;
124            self.fs.sync_dir(to).map_err(fail)?;
125            if let Some(parent) = to.parent() {
126                self.fs.sync_dir(parent).map_err(fail)?;
127            }
128            return Ok(Backup {
129                tenants: listed.len() as u64,
130                versions,
131                tries,
132            });
133        }
134        Err(BackupError::KeptChanging)
135    }
136
137    /// Whether server.sqlite, as it is now, lists the tenant.
138    fn listed(&self, id: TenantId) -> Result<bool, BackupError> {
139        use rusqlite::OptionalExtension;
140        Ok(self
141            .conn()
142            .query_row("SELECT 1 FROM tenant WHERE id = ?1", [id.0], |_| Ok(()))
143            .optional()
144            .map_err(|_| BackupError::Io)?
145            .is_some())
146    }
147}
148
149/// The tenants the server.sqlite copy at `path` lists.
150fn tenants_in(path: &Path, key: &crate::envelope::Key) -> Result<Vec<TenantId>, BackupError> {
151    let fail = |_| BackupError::Io;
152    let conn = catalog::open(path, key, "").map_err(|_| BackupError::Io)?;
153    let mut rows = conn
154        .prepare("SELECT id FROM tenant ORDER BY id")
155        .map_err(fail)?;
156    rows.query_map([], |r| r.get::<_, [u8; 16]>(0))
157        .map_err(fail)?
158        .map(|row| row.map(TenantId).map_err(fail))
159        .collect()
160}
161
162fn remove_if_there(path: &Path) -> io::Result<()> {
163    match std::fs::remove_file(path) {
164        Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
165        _ => Ok(()),
166    }
167}
168
169#[cfg(test)]
170mod tests {
171    use std::sync::Arc;
172    use std::sync::atomic::{AtomicUsize, Ordering};
173
174    use super::*;
175    use crate::clock::Clock;
176    use crate::fs::Fs;
177    use crate::ids::TenantId;
178    use crate::store::Caller;
179    use crate::store::testing::vcon;
180    use crate::tenants::Removal;
181    use crate::tenants::tests::{Fixture, fixture};
182
183    /// Two tenants, each holding a vCon.
184    fn two(f: &Fixture) -> [TenantId; 2] {
185        let caller = Caller {
186            key: crate::envelope::Key::generate().expect("key").id(),
187        };
188        let ids = ["acme", "globex"]
189            .map(|name| f.tenants.create(Actor::Cli, name, None).expect("creates"));
190        for (n, id) in ids.iter().enumerate() {
191            let (found, body) = vcon(&format!("01a07da8-c2bb-83e5-b9a2-27900000000{n}"), "held");
192            f.tenants
193                .store(*id)
194                .expect("store")
195                .create(&caller, &found, &body)
196                .expect("create");
197        }
198        ids
199    }
200
201    /// The backup at `to` opened as a registry under the same master key.
202    fn opened(f: &Fixture, to: &Path) -> Tenants {
203        Tenants::open_beside(to, f.master_key(), f.fs.clone(), f.clock.clone())
204            .expect("the backup opens under the master key")
205    }
206
207    fn assert_whole(backup: &Tenants, tenants: usize) {
208        let now = crate::clock::SystemClock.now();
209        let found = backup
210            .examine(crate::retention::Retention::Years(7), now)
211            .expect("examines");
212        assert_eq!(found.tenants.len(), tenants);
213        for tenant in &found.tenants {
214            let store = tenant
215                .store
216                .as_ref()
217                .expect("each catalog opens under its key");
218            assert!(store.damaged.is_empty(), "{:?}", store.damaged);
219            assert!(store.leftovers.is_empty(), "{:?}", store.leftovers);
220            assert_eq!(store.versions, 1);
221        }
222    }
223
224    fn records(f: &Fixture, action: &str) -> Vec<serde_json::Value> {
225        f.tenants
226            .admin_records()
227            .into_iter()
228            .filter(|r| r["action"] == action)
229            .collect()
230    }
231
232    #[test]
233    fn a_backup_opens_whole_under_the_master_key_and_is_recorded() {
234        let f = fixture();
235        two(&f);
236        let to = f.dir.path().join("backup");
237        let done = f.tenants.backup(Actor::Cli, &to).expect("backs up");
238        assert_eq!(
239            done,
240            Backup {
241                tenants: 2,
242                versions: 2,
243                tries: 1
244            }
245        );
246        let backup = opened(&f, &to);
247        assert_whole(&backup, 2);
248        use std::os::unix::fs::PermissionsExt;
249        let mode = std::fs::metadata(&to)
250            .expect("the backup")
251            .permissions()
252            .mode()
253            & 0o777;
254        assert_eq!(
255            mode, 0o700,
256            "only the service account may enter the backup: {mode:o}"
257        );
258        let names: Vec<String> = backup
259            .list(Actor::Cli)
260            .expect("lists")
261            .into_iter()
262            .map(|t| t.name)
263            .collect();
264        assert_eq!(names, ["acme", "globex"]);
265        let recorded = records(&f, "backup");
266        assert_eq!(recorded.len(), 1);
267        assert_eq!(recorded[0]["status"], 200);
268    }
269
270    #[test]
271    fn a_backup_survives_a_crash_once_it_returns() {
272        let f = fixture();
273        two(&f);
274        let to = f.dir.path().join("backup");
275        f.tenants.backup(Actor::Cli, &to).expect("backs up");
276        f.fs.crash();
277        assert_eq!(
278            f.fs.list(&to.join("tenants"))
279                .expect("the backup is there")
280                .len(),
281            2
282        );
283        assert_whole(&opened(&f, &to), 2);
284    }
285
286    #[test]
287    fn a_path_already_there_is_refused_and_recorded() {
288        let f = fixture();
289        two(&f);
290        let to = f.dir.path().join("backup");
291        f.fs.create_dir_all(&to).expect("dir");
292        assert_eq!(f.tenants.backup(Actor::Cli, &to), Err(BackupError::Exists));
293        assert!(
294            f.fs.list(&to).expect("list").is_empty(),
295            "nothing is written"
296        );
297        assert_eq!(records(&f, "backup")[0]["status"], 409);
298    }
299
300    #[test]
301    fn a_tenant_removed_after_the_registry_is_copied_starts_the_backup_again() {
302        let f = fixture();
303        let [acme, _] = two(&f);
304        let calls = Arc::new(AtomicUsize::new(0));
305        let seen = calls.clone();
306        *f.tenants.hooks.registry_copied.lock().expect("hook") =
307            Some(Box::new(move |tenants: &Tenants| {
308                if seen.fetch_add(1, Ordering::SeqCst) == 0 {
309                    tenants
310                        .remove(Actor::Cli, acme, Removal::Force)
311                        .expect("removes");
312                }
313            }));
314        let to = f.dir.path().join("backup");
315        let done = f.tenants.backup(Actor::Cli, &to).expect("backs up");
316        assert_eq!(
317            done,
318            Backup {
319                tenants: 1,
320                versions: 1,
321                tries: 2
322            }
323        );
324        assert_whole(&opened(&f, &to), 1);
325        let dirs = f.fs.list(&to.join("tenants")).expect("list");
326        assert_eq!(
327            dirs.len(),
328            1,
329            "no directory is left for the removed tenant: {dirs:?}"
330        );
331    }
332
333    #[test]
334    fn a_tenant_removed_while_its_data_is_copied_starts_the_backup_again() {
335        let f = fixture();
336        let [acme, _] = two(&f);
337        // Another registry on the same directory, as the server would be,
338        // removes the tenant while this one copies its files: after the
339        // tenant's catalog is copied, before its object files are.
340        let server =
341            Tenants::open_beside(f.dir.path(), f.master_key(), f.fs.clone(), f.clock.clone())
342                .expect("a second registry");
343        let once = std::sync::Mutex::new(Some(server));
344        f.tenants
345            .store(acme)
346            .expect("store")
347            .on_backup_copied(Box::new(move |_: &crate::store::Store| {
348                if let Some(server) = once.lock().expect("once").take() {
349                    server
350                        .remove(Actor::Cli, acme, Removal::Force)
351                        .expect("removes");
352                }
353            }));
354        let to = f.dir.path().join("backup");
355        let done = f.tenants.backup(Actor::Cli, &to).expect("backs up");
356        assert_eq!(done.tenants, 1);
357        assert_eq!(done.tries, 2);
358        assert_whole(&opened(&f, &to), 1);
359        let dirs = f.fs.list(&to.join("tenants")).expect("list");
360        assert_eq!(
361            dirs.len(),
362            1,
363            "the removed tenant's partial copy is gone: {dirs:?}"
364        );
365    }
366
367    #[test]
368    fn a_registry_that_keeps_changing_fails_after_its_tries_and_is_recorded() {
369        let f = fixture();
370        let [acme, _] = two(&f);
371        let calls = Arc::new(AtomicUsize::new(0));
372        let seen = calls.clone();
373        let listed = std::sync::Mutex::new(acme);
374        *f.tenants.hooks.registry_copied.lock().expect("hook") =
375            Some(Box::new(move |tenants: &Tenants| {
376                // Each try: a tenant the copy lists is removed before its
377                // data is copied, and a new one is made, which the next
378                // copy lists.
379                let n = seen.fetch_add(1, Ordering::SeqCst);
380                let mut listed = listed.lock().expect("listed");
381                tenants
382                    .remove(Actor::Cli, *listed, Removal::Force)
383                    .expect("removes");
384                *listed = tenants
385                    .create(Actor::Cli, &format!("brief {n}"), None)
386                    .expect("creates");
387            }));
388        let to = f.dir.path().join("backup");
389        assert_eq!(
390            f.tenants.backup(Actor::Cli, &to),
391            Err(BackupError::KeptChanging)
392        );
393        assert_eq!(calls.load(Ordering::SeqCst), crate::store::TRIES);
394        assert_eq!(records(&f, "backup")[0]["status"], 503);
395    }
396}
vcon_server/tenants/backup.rs

vcon_server/tenants/
backup.rs
