obnam_benchmark/

suite.rs

use crate::builder::{ObnamBuilder, ObnamBuilderError};
use crate::client::{ObnamClient, ObnamClientError};
use crate::daemon::DaemonManager;
use crate::junk::junk;
use crate::result::{Measurement, OpMeasurements, Operation};
use crate::server::{ObnamServer, ObnamServerError};
use crate::specification::{Create, FileCount};
use crate::step::Step;
use crate::summain::{summain, SummainError};
use log::{debug, error, info};
use std::collections::HashMap;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Instant;
use tempfile::{tempdir, TempDir};
use walkdir::WalkDir;

/// A running benchmark suite.
///
/// This manages temporary data created for the benchmarks, and
/// executes individual steps in the suite.
pub struct Suite {
    client: PathBuf,
    server: PathBuf,
    manager: DaemonManager,
    benchmark: Option<Benchmark>,
}

/// Possible errors from running a benchmark suite.
#[derive(Debug, thiserror::Error)]
pub enum SuiteError {
    /// Error building Obnam.
    #[error(transparent)]
    Build(ObnamBuilderError),

    /// Error creating a temporary directory.
    #[error(transparent)]
    TempDir(#[from] std::io::Error),

    /// File creation failed.
    #[error("Failed to create file {0}: {1}")]
    CreateFile(PathBuf, std::io::Error),

    /// Error from counting files.
    #[error("Error counting files in {0}: {1}")]
    FileCount(PathBuf, walkdir::Error),

    /// Error looking up file metadata.
    #[error("Error looking up file metadata: {0}: {1}")]
    FileMeta(PathBuf, walkdir::Error),

    /// Error removing restored data.
    #[error("Error removing temporary directory: {0}: {1}")]
    RemoveRestored(PathBuf, std::io::Error),

    /// Error using an Obnam client.
    #[error(transparent)]
    Client(#[from] ObnamClientError),

    /// Error managing an Obnam server.
    #[error(transparent)]
    Server(#[from] ObnamServerError),

    /// Suite already has a manifest with a given id.
    #[error("Suite already has manifest {0}: this is a bug")]
    ManifestExists(usize),

    /// Suite doesn't have a manifest with a given id.
    #[error("Suite doesn't have a manifest {0}: this is a bug")]
    ManifestMissing(usize),

    /// Manifests are not identical.
    #[error("Manifests {0} and {1} are not identical, as expected")]
    ManifestsDiffer(usize, usize),

    /// Error running summain.
    #[error(transparent)]
    Summain(SummainError),
}

impl Suite {
    pub fn new(builder: &ObnamBuilder) -> Result<Self, SuiteError> {
        Ok(Self {
            client: builder.client_binary().to_path_buf(),
            server: builder.server_binary().to_path_buf(),
            manager: DaemonManager::new(),
            benchmark: None,
        })
    }

    /// Execute one step in the benchmark suite.
    ///
    /// Return a measurement of the step.
    pub fn execute(&mut self, step: &Step) -> Result<OpMeasurements, SuiteError> {
        let time = Instant::now();
        eprintln!("step: {:?}", step);
        let mut om = match step {
            Step::Start(name) => {
                assert!(self.benchmark.is_none());
                let mut benchmark =
                    Benchmark::new(&self.client, &self.server, name, &self.manager)?;
                let om = benchmark.start()?;
                self.benchmark = Some(benchmark);
                om
            }
            Step::Stop(name) => {
                assert!(self.benchmark.is_some());
                assert_eq!(name, self.benchmark.as_ref().unwrap().name());
                let om = self.benchmark.as_mut().unwrap().stop()?;
                self.benchmark = None;
                om
            }
            Step::Create(x) => {
                assert!(self.benchmark.is_some());
                self.benchmark.as_mut().unwrap().create(x)?
            }
            Step::Rename(x) => {
                assert!(self.benchmark.is_some());
                self.benchmark.as_mut().unwrap().rename(x)?
            }
            Step::Delete(x) => {
                assert!(self.benchmark.is_some());
                self.benchmark.as_mut().unwrap().delete(x)?
            }
            Step::Backup(x) => {
                assert!(self.benchmark.is_some());
                self.benchmark.as_mut().unwrap().backup(*x)?
            }
            Step::Restore(x) => {
                assert!(self.benchmark.is_some());
                self.benchmark.as_mut().unwrap().restore(*x)?
            }
            Step::ManifestLive(id) => {
                assert!(self.benchmark.is_some());
                self.benchmark.as_mut().unwrap().manifest_live(*id)?
            }
            Step::ManifestRestored(id) => {
                assert!(self.benchmark.is_some());
                self.benchmark.as_mut().unwrap().manifest_restored(*id)?
            }
            Step::CompareManifests(first, second) => {
                assert!(self.benchmark.is_some());
                self.benchmark
                    .as_mut()
                    .unwrap()
                    .compare_manifests(*first, *second)?
            }
        };

        let t = std::time::Duration::from_millis(10);
        std::thread::sleep(t);

        let ms = time.elapsed().as_millis();
        debug!("step duration was {} ms", ms);
        om.push(Measurement::DurationMs(ms));
        Ok(om)
    }
}

struct Benchmark {
    name: String,
    client: ObnamClient,
    server: ObnamServer,
    live: TempDir,
    restored: TempDir,
    gen_ids: HashMap<usize, String>,
    manifests: HashMap<usize, String>,
}

impl Benchmark {
    fn new(
        client: &Path,
        server: &Path,
        name: &str,
        manager: &DaemonManager,
    ) -> Result<Self, SuiteError> {
        let server = ObnamServer::new(server, manager)?;
        let live = tempdir().map_err(SuiteError::TempDir)?;
        let restored = tempdir().map_err(SuiteError::TempDir)?;
        let client = ObnamClient::new(client, server.url(), live.path().to_path_buf())?;
        Ok(Self {
            name: name.to_string(),
            client,
            server,
            live,
            restored,
            gen_ids: HashMap::new(),
            manifests: HashMap::new(),
        })
    }

    fn name(&self) -> &str {
        &self.name
    }

    fn live(&self) -> PathBuf {
        self.live.path().to_path_buf()
    }

    fn restored(&self) -> PathBuf {
        self.restored.path().join("restored")
    }

    fn start(&mut self) -> Result<OpMeasurements, SuiteError> {
        info!("starting benchmark {}", self.name());
        self.client
            .run(&["init", "--insecure-passphrase=hunter2"])?;
        Ok(OpMeasurements::new(self.name(), Operation::Start))
    }

    fn stop(&mut self) -> Result<OpMeasurements, SuiteError> {
        info!("ending benchmark {}", self.name);
        self.server.stop();
        Ok(OpMeasurements::new(self.name(), Operation::Stop))
    }

    fn create(&mut self, create: &Create) -> Result<OpMeasurements, SuiteError> {
        let root = self.live();
        info!(
            "creating {} files of {} bytes each in {}",
            create.files,
            create.file_size,
            root.display()
        );

        for i in 0..create.files {
            let filename = root.join(format!("{}", i));
            let mut f =
                File::create(&filename).map_err(|err| SuiteError::CreateFile(filename, err))?;
            junk(&mut f, create.file_size)?;
        }

        Ok(OpMeasurements::new(self.name(), Operation::Create))
    }

    fn rename(&mut self, count: &FileCount) -> Result<OpMeasurements, SuiteError> {
        info!("renaming {} test data files", count.files);
        Ok(OpMeasurements::new(self.name(), Operation::Rename))
    }

    fn delete(&mut self, count: &FileCount) -> Result<OpMeasurements, SuiteError> {
        info!("deleting {} test data files", count.files);
        Ok(OpMeasurements::new(self.name(), Operation::Delete))
    }

    fn backup(&mut self, n: usize) -> Result<OpMeasurements, SuiteError> {
        info!("making backup {} in benchmark {}", n, self.name());
        self.client.run(&["backup"])?;
        let gen_id = self
            .client
            .run(&["resolve", "latest"])?
            .strip_suffix('\n')
            .or(Some(""))
            .unwrap()
            .to_string();
        debug!("backed up generation {}", gen_id);
        self.gen_ids.insert(n, gen_id);
        let mut om = OpMeasurements::new(self.name(), Operation::Backup);
        let stats = filestats(&self.live())?;
        om.push(Measurement::TotalFiles(stats.count));
        om.push(Measurement::TotalData(stats.size));
        Ok(om)
    }

    fn restore(&mut self, n: usize) -> Result<OpMeasurements, SuiteError> {
        info!("restoring backup {} in benchmark {}", n, self.name());
        debug!("first removing all data from restore directory");
        let restored = self.restored();
        if restored.exists() {
            std::fs::remove_dir_all(&restored)
                .map_err(|err| SuiteError::RemoveRestored(restored, err))?;
        }
        let gen_id = self.gen_ids.get(&n).unwrap();
        let path = self.restored().display().to_string();
        self.client.run(&["restore", gen_id, &path])?;
        Ok(OpMeasurements::new(self.name(), Operation::Restore))
    }

    fn manifest_live(&mut self, id: usize) -> Result<OpMeasurements, SuiteError> {
        info!("make manifest {} of current test data", id);
        if self.manifests.contains_key(&id) {
            return Err(SuiteError::ManifestExists(id));
        }
        let m = summain(self.live.path()).map_err(SuiteError::Summain)?;
        self.manifests.insert(id, m);
        Ok(OpMeasurements::new(self.name(), Operation::ManifestLive))
    }

    fn manifest_restored(&mut self, id: usize) -> Result<OpMeasurements, SuiteError> {
        info!("make manifest {} of latest restored data", id);
        if self.manifests.contains_key(&id) {
            return Err(SuiteError::ManifestExists(id));
        }
        debug!("self.restored()={}", self.restored().display());
        let restored = format!(
            "{}{}",
            self.restored().display(),
            self.live.path().display()
        );
        let restored = Path::new(&restored);
        debug!("restored directory is {}", restored.display());
        let m = summain(restored).map_err(SuiteError::Summain)?;
        self.manifests.insert(id, m);
        Ok(OpMeasurements::new(
            self.name(),
            Operation::ManifestRestored,
        ))
    }

    fn compare_manifests(
        &mut self,
        first: usize,
        second: usize,
    ) -> Result<OpMeasurements, SuiteError> {
        info!("compare manifests {} and {}", first, second);
        let m1 = self.manifest(first)?;
        let m2 = self.manifest(second)?;
        if m1 != m2 {
            error!("first manifest:\n{}", m1);
            error!("second manifest:\n{}", m2);
            return Err(SuiteError::ManifestsDiffer(first, second));
        }
        Ok(OpMeasurements::new(self.name(), Operation::CompareManiests))
    }

    fn manifest(&self, id: usize) -> Result<String, SuiteError> {
        if let Some(m) = self.manifests.get(&id) {
            Ok(m.clone())
        } else {
            Err(SuiteError::ManifestMissing(id))
        }
    }
}

#[derive(Debug, Default)]
struct FileStats {
    count: u64,
    size: u64,
}

fn filestats(dirname: &Path) -> Result<FileStats, SuiteError> {
    let mut stats = FileStats::default();
    for e in WalkDir::new(dirname) {
        let e = e.map_err(|err| SuiteError::FileCount(dirname.to_path_buf(), err))?;
        stats.count += 1;
        stats.size += e
            .metadata()
            .map_err(|err| SuiteError::FileMeta(e.path().to_path_buf(), err))?
            .len();
    }
    Ok(stats)
}