vcon_server/tenants/
backup.rs

1//! A copy of the whole data directory, made while a server may be running:
2//! server.sqlite with SQLite's online backup API, then each tenant it lists,
3//! as `Store::backup_to` copies one.
4
5use std::collections::BTreeSet;
6use std::io;
7use std::path::Path;
8
9use super::{TenantError, Tenants};
10use crate::audit::{Action, Actor, Entry};
11use crate::ids::TenantId;
12use crate::store::{self, TRIES};
13
14/// What `backup` copied.
15#[derive(Debug, Clone, Copy, PartialEq, Eq)]
16pub struct Backup {
17    pub tenants: u64,
18    pub versions: u64,
19    /// How many times server.sqlite was copied.
20    pub tries: usize,
21}
22
23/// Why a backup was refused or failed.
24#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25pub enum BackupError {
26    /// Something is already at the backup's path.
27    Exists,
28    /// Tenants were removed, or versions purged, after each try's copy.
29    KeptChanging,
30    /// A file could not be read, written or synced, or a catalog could not
31    /// be copied or read.
32    Io,
33    /// The admin log could not record the backup.
34    AuditFailed,
35}
36
37impl Tenants {
38    /// Copies the data directory into `to`, which must not exist, and
39    /// records the run in the admin log with its outcome.
40    ///
41    /// Each try copies server.sqlite in one step inside one read
42    /// transaction, then each tenant that copy lists, as `Store::backup_to`
43    /// copies one. A tenant the copy lists that is gone, or whose versions
44    /// kept being purged, was removed meanwhile, and the next try copies
45    /// server.sqlite again. Tenant directories the final copy does not
46    /// list are removed. Tenants are copied one after another, each as it
47    /// was at its own copy.
48    pub fn backup(&self, who: Actor, to: &Path) -> Result<Backup, BackupError> {
49        let done = self.copy_all(to);
50        let status = match done {
51            Ok(_) => 200,
52            Err(BackupError::Exists) => 409,
53            Err(BackupError::KeptChanging) => 503,
54            Err(BackupError::Io | BackupError::AuditFailed) => 500,
55        };
56        self.record_alone(&Entry {
57            actor: who,
58            action: Action::Backup,
59            target: None,
60            version: None,
61            status,
62        })
63        .map_err(|_| BackupError::AuditFailed)?;
64        done
65    }
66
67    fn copy_all(&self, to: &Path) -> Result<Backup, BackupError> {
68        let fail = |_| BackupError::Io;
69        match self.fs.list(to) {
70            Err(e) if e.kind() == io::ErrorKind::NotFound => {}
71            Ok(_) => return Err(BackupError::Exists),
72            Err(e) if e.kind() == io::ErrorKind::NotADirectory => return Err(BackupError::Exists),
73            Err(_) => return Err(BackupError::Io),
74        }
75        let tenants = to.join("tenants");
76        self.fs.create_dir_all(&tenants).map_err(fail)?;
77        // Only the service account may enter the copy, as with the data
78        // directory, which is restored by copying this one with its modes.
79        {
80            use std::os::unix::fs::PermissionsExt;
81            std::fs::set_permissions(to, std::fs::Permissions::from_mode(crate::fs::DIR_MODE))
82                .map_err(fail)?;
83        }
84        let registry = to.join("server.sqlite");
85        'tries: for tries in 1..=TRIES {
86            for file in ["server.sqlite", "server.sqlite-wal", "server.sqlite-shm"] {
87                remove_if_there(&to.join(file)).map_err(fail)?;
88            }
89            let listed = self
90                .registry
91                .backup_into(&registry)
92                .map_err(|_| BackupError::Io)?;
93            #[cfg(test)]
94            if let Some(hook) = self.hooks.registry_copied.lock().expect("hook").as_mut() {
95                hook(self);
96            }
97            let mut versions = 0;
98            for id in &listed {
99                let copied = match self.store(*id) {
100                    Ok(store) => store.backup_to(&tenants.join(id.to_string())),
101                    Err(TenantError::NotFound | TenantError::Closing) => continue 'tries,
102                    Err(_) => return Err(BackupError::Io),
103                };
104                match copied {
105                    Ok(copied) => versions += copied.versions,
106                    // A tenant removed while it was copied: its files went
107                    // with it, so no try of its own could finish.
108                    Err(store::BackupError::KeptChanging) if !self.listed(*id)? => {
109                        self.stores().remove(id);
110                        continue 'tries;
111                    }
112                    Err(store::BackupError::KeptChanging) => return Err(BackupError::KeptChanging),
113                    Err(store::BackupError::Io) => return Err(BackupError::Io),
114                }
115            }
116            let names: BTreeSet<String> = listed.iter().map(ToString::to_string).collect();
117            for name in self.fs.list(&tenants).map_err(fail)? {
118                let name = name.into_string().unwrap_or_default();
119                if !names.contains(&name) {
120                    self.fs.remove_dir_all(&tenants.join(&name)).map_err(fail)?;
121                }
122            }
123            self.fs.sync_dir(&tenants).map_err(fail)?;
124            self.fs.sync_dir(to).map_err(fail)?;
125            if let Some(parent) = to.parent() {
126                self.fs.sync_dir(parent).map_err(fail)?;
127            }
128            return Ok(Backup {
129                tenants: listed.len() as u64,
130                versions,
131                tries,
132            });
133        }
134        Err(BackupError::KeptChanging)
135    }
136
137    /// Whether server.sqlite, as it is now, lists the tenant.
138    fn listed(&self, id: TenantId) -> Result<bool, BackupError> {
139        let mut tx = self.registry.snapshot().map_err(|_| BackupError::Io)?;
140        Ok(tx.tenant(id).map_err(|_| BackupError::Io)?.is_some())
141    }
142}
143
144fn remove_if_there(path: &Path) -> io::Result<()> {
145    match std::fs::remove_file(path) {
146        Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e),
147        _ => Ok(()),
148    }
149}
150
151#[cfg(test)]
152mod tests {
153    use std::sync::Arc;
154    use std::sync::atomic::{AtomicUsize, Ordering};
155
156    use super::*;
157    use crate::clock::Clock;
158    use crate::fs::Fs;
159    use crate::ids::TenantId;
160    use crate::store::Caller;
161    use crate::store::testing::vcon;
162    use crate::tenants::Removal;
163    use crate::tenants::tests::{Fixture, fixture};
164
165    /// Two tenants, each holding a vCon.
166    fn two(f: &Fixture) -> [TenantId; 2] {
167        let caller = Caller {
168            key: crate::envelope::Key::generate().expect("key").id(),
169        };
170        let ids = ["acme", "globex"]
171            .map(|name| f.tenants.create(Actor::Cli, name, None).expect("creates"));
172        for (n, id) in ids.iter().enumerate() {
173            let (found, body) = vcon(&format!("01a07da8-c2bb-83e5-b9a2-27900000000{n}"), "held");
174            f.tenants
175                .store(*id)
176                .expect("store")
177                .create(&caller, &found, &body)
178                .expect("create");
179        }
180        ids
181    }
182
183    /// The backup at `to` opened as a registry under the same master key.
184    fn opened(f: &Fixture, to: &Path) -> Tenants {
185        Tenants::open_beside(to, f.master_key(), f.fs.clone(), f.clock.clone())
186            .expect("the backup opens under the master key")
187    }
188
189    /// Checks the backup `backup` holds `tenants` tenants, each whole with
190    /// its one version, examined at the fixture's time.
191    fn assert_whole(backup: &Tenants, tenants: usize, now: time::OffsetDateTime) {
192        let found = backup
193            .examine(crate::retention::Retention::Years(7), now)
194            .expect("examines");
195        assert_eq!(
196            found.tenants.len(),
197            tenants,
198            "the backup holds every tenant the registry did"
199        );
200        for tenant in &found.tenants {
201            let store = tenant
202                .store
203                .as_ref()
204                .expect("each catalog opens under its key");
205            assert!(
206                store.damaged.is_empty(),
207                "every copied version opens: {:?}",
208                store.damaged
209            );
210            assert!(
211                store.leftovers.is_empty(),
212                "the copy holds no file its catalog does not name: {:?}",
213                store.leftovers
214            );
215            assert_eq!(store.versions, 1, "each tenant stored one version");
216        }
217    }
218
219    fn records(f: &Fixture, action: &str) -> Vec<serde_json::Value> {
220        f.tenants
221            .admin_records()
222            .into_iter()
223            .filter(|r| r["action"] == action)
224            .collect()
225    }
226
227    #[test]
228    fn a_backup_opens_whole_under_the_master_key_and_is_recorded() {
229        let f = fixture();
230        two(&f);
231        let to = f.dir.path().join("backup");
232        let done = f.tenants.backup(Actor::Cli, &to).expect("backs up");
233        assert_eq!(
234            done,
235            Backup {
236                tenants: 2,
237                versions: 2,
238                tries: 1
239            }
240        );
241        let backup = opened(&f, &to);
242        assert_whole(&backup, 2, f.clock.now());
243        use std::os::unix::fs::PermissionsExt;
244        let mode = std::fs::metadata(&to)
245            .expect("the backup")
246            .permissions()
247            .mode()
248            & 0o777;
249        assert_eq!(
250            mode, 0o700,
251            "only the service account may enter the backup: {mode:o}"
252        );
253        let names: Vec<String> = backup
254            .list(Actor::Cli)
255            .expect("lists")
256            .into_iter()
257            .map(|t| t.name)
258            .collect();
259        assert_eq!(names, ["acme", "globex"]);
260        let recorded = records(&f, "backup");
261        assert_eq!(recorded.len(), 1);
262        assert_eq!(recorded[0]["status"], 200);
263    }
264
265    #[test]
266    fn a_backup_survives_a_crash_once_it_returns() {
267        let f = fixture();
268        two(&f);
269        let to = f.dir.path().join("backup");
270        f.tenants.backup(Actor::Cli, &to).expect("backs up");
271        f.fs.crash();
272        assert_eq!(
273            f.fs.list(&to.join("tenants"))
274                .expect("the backup is there")
275                .len(),
276            2
277        );
278        assert_whole(&opened(&f, &to), 2, f.clock.now());
279    }
280
281    #[test]
282    fn a_path_already_there_is_refused_and_recorded() {
283        let f = fixture();
284        two(&f);
285        let to = f.dir.path().join("backup");
286        f.fs.create_dir_all(&to).expect("dir");
287        assert_eq!(f.tenants.backup(Actor::Cli, &to), Err(BackupError::Exists));
288        assert!(
289            f.fs.list(&to).expect("list").is_empty(),
290            "nothing is written"
291        );
292        assert_eq!(records(&f, "backup")[0]["status"], 409);
293    }
294
295    #[test]
296    fn a_tenant_removed_after_the_registry_is_copied_starts_the_backup_again() {
297        let f = fixture();
298        let [acme, _] = two(&f);
299        let calls = Arc::new(AtomicUsize::new(0));
300        let seen = calls.clone();
301        *f.tenants.hooks.registry_copied.lock().expect("hook") =
302            Some(Box::new(move |tenants: &Tenants| {
303                if seen.fetch_add(1, Ordering::SeqCst) == 0 {
304                    tenants
305                        .remove(Actor::Cli, acme, Removal::Force)
306                        .expect("removes");
307                }
308            }));
309        let to = f.dir.path().join("backup");
310        let done = f.tenants.backup(Actor::Cli, &to).expect("backs up");
311        assert_eq!(
312            done,
313            Backup {
314                tenants: 1,
315                versions: 1,
316                tries: 2
317            }
318        );
319        assert_whole(&opened(&f, &to), 1, f.clock.now());
320        let dirs = f.fs.list(&to.join("tenants")).expect("list");
321        assert_eq!(
322            dirs.len(),
323            1,
324            "no directory is left for the removed tenant: {dirs:?}"
325        );
326    }
327
328    #[test]
329    fn a_tenant_removed_while_its_data_is_copied_starts_the_backup_again() {
330        let f = fixture();
331        let [acme, _] = two(&f);
332        // Another registry on the same directory, as the server would be,
333        // removes the tenant while this one copies its files: after the
334        // tenant's catalog is copied, before its object files are.
335        let server =
336            Tenants::open_beside(f.dir.path(), f.master_key(), f.fs.clone(), f.clock.clone())
337                .expect("a second registry");
338        let once = std::sync::Mutex::new(Some(server));
339        f.tenants
340            .store(acme)
341            .expect("store")
342            .on_backup_copied(Box::new(move |_: &crate::store::Store| {
343                if let Some(server) = once.lock().expect("once").take() {
344                    server
345                        .remove(Actor::Cli, acme, Removal::Force)
346                        .expect("removes");
347                }
348            }));
349        let to = f.dir.path().join("backup");
350        let done = f.tenants.backup(Actor::Cli, &to).expect("backs up");
351        assert_eq!(
352            done,
353            Backup {
354                tenants: 1,
355                versions: 1,
356                tries: 2
357            },
358            "the second try copies the one tenant left and its one version"
359        );
360        assert_whole(&opened(&f, &to), 1, f.clock.now());
361        let dirs = f.fs.list(&to.join("tenants")).expect("list");
362        assert_eq!(
363            dirs.len(),
364            1,
365            "the removed tenant's partial copy is gone: {dirs:?}"
366        );
367    }
368
369    #[test]
370    fn a_registry_that_keeps_changing_fails_after_its_tries_and_is_recorded() {
371        let f = fixture();
372        let [acme, _] = two(&f);
373        let calls = Arc::new(AtomicUsize::new(0));
374        let seen = calls.clone();
375        let listed = std::sync::Mutex::new(acme);
376        *f.tenants.hooks.registry_copied.lock().expect("hook") =
377            Some(Box::new(move |tenants: &Tenants| {
378                // Each try: a tenant the copy lists is removed before its
379                // data is copied, and a new one is made, which the next
380                // copy lists.
381                let n = seen.fetch_add(1, Ordering::SeqCst);
382                let mut listed = listed.lock().expect("listed");
383                tenants
384                    .remove(Actor::Cli, *listed, Removal::Force)
385                    .expect("removes");
386                *listed = tenants
387                    .create(Actor::Cli, &format!("brief {n}"), None)
388                    .expect("creates");
389            }));
390        let to = f.dir.path().join("backup");
391        assert_eq!(
392            f.tenants.backup(Actor::Cli, &to),
393            Err(BackupError::KeptChanging)
394        );
395        assert_eq!(calls.load(Ordering::SeqCst), crate::store::TRIES);
396        assert_eq!(records(&f, "backup")[0]["status"], 503);
397    }
398}
vcon_server/tenants/backup.rs

vcon_server/tenants/
backup.rs
