glean-core 70.0.0

mod common;
use std::fs;
use std::os::unix::fs::MetadataExt;

use crate::common::*;

use glean_core::metrics::*;
use glean_core::CommonMetricData;
use glean_core::Lifetime;
use rkv::{Rkv, StoreOptions};
use uuid::uuid;

static RKV_DATABASE: &[u8] = include_bytes!("77ca0472-5124-4f6b-971d-4a2a928fb158.safe.bin");
// This database is based on a submitted `metrics` ping from a real client.
// Only metrics for that ping and a bare minimum of client info data is added,
// a total of 61 metrics.
// This is realistic enough to cause size differences in the non-vacuumed/vacuumed SQLite
// databases after migration.
static FILLED_RKV_DATABASE: &[u8] = include_bytes!("filled-rkv.data.safe.bin");

fn clientid_metric() -> UuidMetric {
    UuidMetric::new(CommonMetricData {
        name: "client_id".into(),
        category: "".into(),
        send_in_pings: vec!["glean_client_info".into()],
        lifetime: Lifetime::User,
        ..Default::default()
    })
}

/// Copy of parts of `DatabaseMetrics` in `glean-core/src/internal_metrics.rs` for ease of testing.
struct MigrationMetrics {
    metrics_in_sqlite: CounterMetric,
    migrated_metrics: CounterMetric,
    failed_metrics: CounterMetric,
    migration_duration: TimingDistributionMetric,
    migration_error: CounterMetric,
}

impl MigrationMetrics {
    fn new() -> Self {
        Self {
            migrated_metrics: CounterMetric::new(CommonMetricData {
                name: "migrated_metrics".into(),
                category: "glean.migration".into(),
                send_in_pings: vec!["metrics".into(), "health".into()],
                lifetime: Lifetime::Ping,
                ..Default::default()
            }),

            metrics_in_sqlite: CounterMetric::new(CommonMetricData {
                name: "metrics_in_sqlite".into(),
                category: "glean.migration".into(),
                send_in_pings: vec!["metrics".into(), "health".into()],
                lifetime: Lifetime::Ping,
                ..Default::default()
            }),

            failed_metrics: CounterMetric::new(CommonMetricData {
                name: "failed_metrics".into(),
                category: "glean.migration".into(),
                send_in_pings: vec!["metrics".into(), "health".into()],
                lifetime: Lifetime::Ping,
                ..Default::default()
            }),

            migration_duration: TimingDistributionMetric::new(
                CommonMetricData {
                    name: "migration_duration".into(),
                    category: "glean.migration".into(),
                    send_in_pings: vec!["metrics".into(), "health".into()],
                    lifetime: Lifetime::Ping,
                    ..Default::default()
                },
                TimeUnit::Millisecond,
            ),
            migration_error: CounterMetric::new(CommonMetricData {
                name: "error".into(),
                category: "glean.migration".into(),
                send_in_pings: vec!["metrics".into(), "health".into()],
                lifetime: Lifetime::Ping,
                ..Default::default()
            }),
        }
    }
}

#[test]
fn migration_succeeds() {
    let temp = tempfile::tempdir().unwrap();
    let db_path = temp.path().join("db");
    fs::create_dir_all(&db_path).unwrap();

    let safe_bin = db_path.join("data.safe.bin");
    // File has been generated from essentially:
    //
    // ```rust
    // let tmpname = PathBuf::new("/tmp/glean-fc");
    // let cfg = ConfigurationBuilder::new(true, tmpname.clone(), "glean-fc")
    //     .with_server_endpoint("invalid-test-host")
    //     .with_use_core_mps(false)
    //     .build();
    // glean::initialize(cfg, client_info);
    // glean::shutdown();
    // ```
    //
    // All ping-specific metrics have been removed.
    // Only client_info metrics are migrated, including the client ID.
    fs::write(safe_bin, RKV_DATABASE).unwrap();
    let exp_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158");

    let (glean, _temp) = new_glean(Some(temp));

    let client_id = clientid_metric().get_value(&glean, None).unwrap();
    assert_eq!(exp_client_id, client_id);

    let metrics = MigrationMetrics::new();
    assert_eq!(Some(13), metrics.migrated_metrics.get_value(&glean, None));
    assert_eq!(Some(13), metrics.metrics_in_sqlite.get_value(&glean, None));
    assert_eq!(None, metrics.failed_metrics.get_value(&glean, None));
    assert!(metrics.migration_duration.get_value(&glean, None).is_some());
    assert_eq!(None, metrics.migration_error.get_value(&glean, None));
}

#[test]
fn migration_skipped_if_database_exists() {
    let (first_client_id, temp) = {
        let (glean, temp) = new_glean(None);
        let client_id = clientid_metric().get_value(&glean, None).unwrap();
        drop(glean);
        (client_id, temp)
    };

    let safe_bin = temp.path().join("db").join("data.safe.bin");
    fs::write(
        &safe_bin,
        include_bytes!("77ca0472-5124-4f6b-971d-4a2a928fb158.safe.bin"),
    )
    .unwrap();
    let rkv_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158");

    let (glean, _temp) = new_glean(Some(temp));

    let client_id = clientid_metric().get_value(&glean, None).unwrap();
    assert_eq!(
        first_client_id, client_id,
        "Client ID should be the one first generated"
    );
    assert_ne!(
        rkv_client_id, client_id,
        "Client ID should not be one from the Rkv database"
    );
    assert!(safe_bin.exists(), "Rkv file should not have been deleted");

    let metrics = MigrationMetrics::new();
    assert_eq!(None, metrics.migrated_metrics.get_value(&glean, None));
    assert_eq!(None, metrics.metrics_in_sqlite.get_value(&glean, None));
    assert_eq!(None, metrics.failed_metrics.get_value(&glean, None));
    assert_eq!(None, metrics.migration_duration.get_value(&glean, None));
    assert_eq!(None, metrics.migration_error.get_value(&glean, None));
}

#[test]
fn migration_succeeds_with_failures() {
    let temp = tempfile::tempdir().unwrap();
    let db_path = temp.path().join("db");
    fs::create_dir_all(&db_path).unwrap();

    let safe_bin = db_path.join("data.safe.bin");
    // Reusing the same database file from above.
    fs::write(safe_bin, RKV_DATABASE).unwrap();
    let exp_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158");

    // Modifying the database to force migration errors.
    {
        let rkv = Rkv::new::<rkv::backend::SafeMode>(&db_path).unwrap();
        let ping_store = rkv
            .open_single(Lifetime::Ping.as_str(), StoreOptions::create())
            .unwrap();
        let mut writer = rkv.write().unwrap();

        let key = "metrics#a.broken.metric";
        let value = rkv::Value::Blob(b"not a value");
        ping_store.put(&mut writer, key, &value).unwrap();

        let key = "baseline#second.broken.metric";
        let value = rkv::Value::I64(31);
        ping_store.put(&mut writer, key, &value).unwrap();

        writer.commit().unwrap();
    }

    let (glean, _temp) = new_glean(Some(temp));

    let client_id = clientid_metric().get_value(&glean, None).unwrap();
    assert_eq!(exp_client_id, client_id);

    let metrics = MigrationMetrics::new();
    assert_eq!(Some(13), metrics.migrated_metrics.get_value(&glean, None));
    assert_eq!(Some(13), metrics.metrics_in_sqlite.get_value(&glean, None));

    // We injected 2 broken metrics.
    assert_eq!(Some(2), metrics.failed_metrics.get_value(&glean, None));

    assert!(metrics.migration_duration.get_value(&glean, None).is_some());
    assert_eq!(None, metrics.migration_error.get_value(&glean, None));
}

#[test]
fn migration_fails() {
    let temp = tempfile::tempdir().unwrap();
    let db_path = temp.path().join("db");
    fs::create_dir_all(&db_path).unwrap();

    let safe_bin = db_path.join("data.safe.bin");
    fs::write(safe_bin, b"\0").unwrap();

    let (glean, _temp) = new_glean(Some(temp));

    let metrics = MigrationMetrics::new();
    assert_eq!(Some(1), metrics.migration_error.get_value(&glean, None));

    assert_eq!(None, metrics.migrated_metrics.get_value(&glean, None));
    assert_eq!(None, metrics.metrics_in_sqlite.get_value(&glean, None));
    assert_eq!(None, metrics.failed_metrics.get_value(&glean, None));
    assert_eq!(None, metrics.migration_duration.get_value(&glean, None));
}

#[test]
fn migration_checkpoints() {
    let temp = tempfile::tempdir().unwrap();
    let db_path = temp.path().join("db");
    fs::create_dir_all(&db_path).unwrap();

    let safe_bin = db_path.join("data.safe.bin");
    // Reusing the same database file from above.
    fs::write(safe_bin, FILLED_RKV_DATABASE).unwrap();
    let exp_client_id = uuid!("3114d9df-9ae3-43a7-83b0-3540c3eba886");

    let (glean, _temp) = new_glean(Some(temp));

    let client_id = clientid_metric().get_value(&glean, None).unwrap();
    assert_eq!(exp_client_id, client_id);

    let metrics = MigrationMetrics::new();
    assert_eq!(Some(61), metrics.migrated_metrics.get_value(&glean, None));
    assert_eq!(Some(61), metrics.metrics_in_sqlite.get_value(&glean, None));

    assert!(metrics.migration_duration.get_value(&glean, None).is_some());
    assert_eq!(None, metrics.migration_error.get_value(&glean, None));

    // Ensure we close the database connection.
    drop(glean);

    let db_file = db_path.join("glean.sqlite");
    let db_file_size = fs::metadata(db_file).unwrap().size();

    // This test is very vague, but it's hard to do better right now.
    //
    // Unvacuumed the database is _smaller_, around 20k bytes, because the migrated data is in the WAL file.
    // Vacuumed & checkpointed the WAL transactions are merged into the database.
    // As of writing that database is at least 9 pages big (9 * 4096 bytes = 36864 bytes).
    // This might grow if we add more metrics.
    // This might shrink if we remove metrics, in which case this test will break and needs adjustement.
    let vacuumed_database_size = 36864;
    assert!(db_file_size >= vacuumed_database_size);
}

#[test]
fn migration_reapplied_after_not_finishing() {
    let temp = {
        let (glean, temp) = new_glean(None);
        drop(glean);

        let db_path = temp.path().join("db").join("glean.sqlite");
        let conn = rusqlite::Connection::open(db_path).unwrap();

        // Let's start with an empty database.
        conn.execute("DELETE FROM telemetry", []).unwrap();

        // Ensure migration isn't marked as done.
        conn.execute("DELETE FROM migration", []).unwrap();
        temp
    };

    let db_path = temp.path().join("db");

    let safe_bin = db_path.join("data.safe.bin");
    // Reusing the same database file from above.
    fs::write(safe_bin, RKV_DATABASE).unwrap();
    let exp_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158");

    let (glean, _temp) = new_glean(Some(temp));

    let client_id = clientid_metric().get_value(&glean, None).unwrap();
    assert_eq!(exp_client_id, client_id);

    let metrics = MigrationMetrics::new();
    assert_eq!(Some(13), metrics.migrated_metrics.get_value(&glean, None));
    assert_eq!(Some(13), metrics.metrics_in_sqlite.get_value(&glean, None));
    assert_eq!(None, metrics.failed_metrics.get_value(&glean, None));

    assert!(metrics.migration_duration.get_value(&glean, None).is_some());
    assert_eq!(None, metrics.migration_error.get_value(&glean, None));
}

#[test]
fn migration_not_reapplied_if_marked_as_finished() {
    let (first_client_id, temp) = {
        let (glean, temp) = new_glean(None);
        let client_id = clientid_metric().get_value(&glean, None).unwrap();
        drop(glean);

        let db_path = temp.path().join("db").join("glean.sqlite");
        let conn = rusqlite::Connection::open(db_path).unwrap();

        // Ensure migration is marked as done.
        conn.execute("DELETE FROM migration", []).unwrap();
        conn.execute("INSERT INTO migration (id, state) VALUES (1, 'done')", [])
            .unwrap();
        (client_id, temp)
    };

    let db_path = temp.path().join("db");

    let safe_bin = db_path.join("data.safe.bin");
    // Reusing the same database file from above.
    fs::write(safe_bin, RKV_DATABASE).unwrap();
    let rkv_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158");

    let (glean, _temp) = new_glean(Some(temp));

    let client_id = clientid_metric().get_value(&glean, None).unwrap();
    assert_ne!(rkv_client_id, client_id);
    assert_eq!(first_client_id, client_id);

    // No migration happened.
    let metrics = MigrationMetrics::new();
    assert_eq!(None, metrics.migrated_metrics.get_value(&glean, None));
    assert_eq!(None, metrics.metrics_in_sqlite.get_value(&glean, None));
    assert_eq!(None, metrics.failed_metrics.get_value(&glean, None));
    assert_eq!(None, metrics.migration_duration.get_value(&glean, None));
    assert_eq!(None, metrics.migration_error.get_value(&glean, None));
}