velesdb-core 7.0.0

//! Tests for `Database::analyze_collection` and `get_collection_stats`.

#![allow(clippy::cast_precision_loss)]

use crate::database::Database;
use crate::distance::DistanceMetric;
use crate::point::Point;
use tempfile::TempDir;

/// Helper: open a database in a temp dir.
fn temp_database() -> (TempDir, Database) {
    let dir = TempDir::new().expect("tempdir");
    let db = Database::open(dir.path()).expect("open database");
    (dir, db)
}

/// Helper: create a collection and insert some points.
fn setup_collection(db: &Database, name: &str, dim: usize, count: u64) {
    db.create_collection(name, dim, DistanceMetric::Cosine)
        .expect("create collection");

    let coll = db.get_vector_collection(name).expect("collection exists");
    let points: Vec<Point> = (1..=count)
        .map(|i| Point {
            id: i,
            vector: vec![i as f32; dim],
            payload: None,
            sparse_vectors: None,
        })
        .collect();
    coll.upsert(points).expect("upsert");
}

// ─────────────────────────────────────────────────────────────────────────────
// analyze_collection
// ─────────────────────────────────────────────────────────────────────────────

#[test]
fn analyze_collection_returns_valid_stats() {
    let (_dir, db) = temp_database();
    setup_collection(&db, "test_stats", 4, 10);

    let stats = db.analyze_collection("test_stats").expect("analyze");
    assert_eq!(stats.total_points, 10);
}

#[test]
fn analyze_collection_nonexistent_returns_error() {
    let (_dir, db) = temp_database();
    let result = db.analyze_collection("nonexistent");
    assert!(result.is_err());
}

#[test]
fn analyze_collection_persists_to_disk() {
    let (dir, db) = temp_database();
    setup_collection(&db, "persist_stats", 4, 5);

    db.analyze_collection("persist_stats").expect("analyze");

    // The stats file should now exist on disk
    let stats_path = dir
        .path()
        .join("persist_stats")
        .join("collection.stats.json");
    assert!(stats_path.exists(), "stats file should be persisted");
}

// ─────────────────────────────────────────────────────────────────────────────
// get_collection_stats round-trip
// ─────────────────────────────────────────────────────────────────────────────

#[test]
fn get_collection_stats_returns_none_before_analyze() {
    let (_dir, db) = temp_database();
    setup_collection(&db, "no_stats", 4, 3);

    let result = db
        .get_collection_stats("no_stats")
        .expect("should not error");
    assert!(result.is_none(), "no stats before analyze");
}

#[test]
fn get_collection_stats_returns_cached_after_analyze() {
    let (_dir, db) = temp_database();
    setup_collection(&db, "cached_stats", 4, 7);

    let original = db.analyze_collection("cached_stats").expect("analyze");

    let cached = db
        .get_collection_stats("cached_stats")
        .expect("get stats")
        .expect("should be Some");
    assert_eq!(cached.total_points, original.total_points);
    assert_eq!(cached.total_points, 7);
}

#[test]
fn get_collection_stats_loads_from_disk() {
    let dir = TempDir::new().expect("tempdir");

    // Open DB, create collection, analyze, then drop DB
    {
        let db = Database::open(dir.path()).expect("open");
        setup_collection(&db, "disk_stats", 4, 8);
        db.analyze_collection("disk_stats").expect("analyze");
    }

    // Re-open DB -- stats should be loadable from disk
    let db2 = Database::open(dir.path()).expect("reopen");
    let loaded = db2
        .get_collection_stats("disk_stats")
        .expect("get stats")
        .expect("should load from disk");
    assert_eq!(loaded.total_points, 8);
}

// ─────────────────────────────────────────────────────────────────────────────
// analyze triggers BFS reorder (issue #377)
// ─────────────────────────────────────────────────────────────────────────────

/// `analyze_collection` must complete without error on a collection that
/// exceeds the BFS reorder threshold (1 000 vectors). Verifies that the
/// reorder_for_locality() call wired into the analyze path doesn't
/// introduce a failure mode for large collections.
#[test]
#[cfg(feature = "persistence")]
fn analyze_collection_reorders_large_collection() {
    let (_dir, db) = temp_database();
    // Use 1 100 vectors so we exceed the REORDER_THRESHOLD of 1 000.
    setup_collection(&db, "reorder_test", 4, 1_100);

    // Should succeed and return valid stats — reorder is a no-error side effect.
    let stats = db
        .analyze_collection("reorder_test")
        .expect("analyze should succeed after reorder");
    assert_eq!(stats.total_points, 1_100);

    // A second analyze should also succeed (reorder is idempotent).
    let stats2 = db
        .analyze_collection("reorder_test")
        .expect("second analyze should also succeed");
    assert_eq!(stats2.total_points, 1_100);
}

// ─────────────────────────────────────────────────────────────────────────────
// Task 8.1: Histogram persistence in collection.stats.json
// ─────────────────────────────────────────────────────────────────────────────

#[test]
fn analyze_collection_persists_histograms() {
    let (dir, db) = temp_database();
    db.create_collection("hist_persist", 4, DistanceMetric::Cosine)
        .expect("create");

    let coll = db
        .get_vector_collection("hist_persist")
        .expect("collection");
    // Insert points with numeric payloads so histograms get built
    let points: Vec<Point> = (1..=100)
        .map(|i| Point {
            id: i,
            vector: vec![i as f32; 4],
            payload: Some(serde_json::json!({"score": i})),
            sparse_vectors: None,
        })
        .collect();
    coll.upsert(points).expect("upsert");

    let stats = db.analyze_collection("hist_persist").expect("analyze");

    // Verify histograms were built for the "score" column
    let has_histogram = stats
        .field_stats
        .get("score")
        .or_else(|| stats.column_stats.get("score"))
        .and_then(|cs| cs.histogram.as_ref())
        .is_some_and(|h| !h.buckets.is_empty());
    assert!(
        has_histogram,
        "histogram should be built for 'score' column"
    );

    // Verify the stats file on disk contains histogram data
    let stats_path = dir
        .path()
        .join("hist_persist")
        .join("collection.stats.json");
    let bytes = std::fs::read(&stats_path).expect("read stats file");
    let json_str = String::from_utf8_lossy(&bytes);
    assert!(
        json_str.contains("\"histogram\""),
        "stats JSON should contain histogram data"
    );
    assert!(
        json_str.contains("\"total_count\""),
        "stats JSON should contain total_count"
    );
}

#[test]
fn histogram_survives_database_reopen() {
    let dir = TempDir::new().expect("tempdir");

    // Phase 1: create, insert, analyze
    {
        let db = Database::open(dir.path()).expect("open");
        db.create_collection("hist_reopen", 4, DistanceMetric::Cosine)
            .expect("create");
        let coll = db.get_vector_collection("hist_reopen").expect("collection");
        let points: Vec<Point> = (1..=50)
            .map(|i| Point {
                id: i,
                vector: vec![i as f32; 4],
                payload: Some(serde_json::json!({"value": i * 10})),
                sparse_vectors: None,
            })
            .collect();
        coll.upsert(points).expect("upsert");
        db.analyze_collection("hist_reopen").expect("analyze");
    }

    // Phase 2: reopen and verify histograms are restored
    let db2 = Database::open(dir.path()).expect("reopen");
    let loaded = db2
        .get_collection_stats("hist_reopen")
        .expect("get stats")
        .expect("stats should load from disk");

    let hist = loaded
        .field_stats
        .get("value")
        .or_else(|| loaded.column_stats.get("value"))
        .and_then(|cs| cs.histogram.as_ref())
        .expect("histogram should be restored after reopen");

    assert!(!hist.buckets.is_empty(), "histogram should have buckets");
    assert!(hist.total_count > 0, "total_count should be positive");
    assert_eq!(
        hist.incremental_updates, 0,
        "fresh analyze has zero updates"
    );
    assert!(!hist.stale, "fresh histogram should not be stale");
}

/// `ANALYZE` must not change which points a query returns.
///
/// The regression this pins: `analyze_collection` runs `reorder_for_locality`
/// above 1 000 points, which renumbers every graph node — and the external-id
/// mappings are keyed by that numbering. Nothing failed when they were left
/// behind: every lookup still resolved, to a different point than the one
/// found. On this fixture the correct `[500, 501, 499, 502, 498]` came back
/// as `[575, 601, 586, 560, 588]` (#2112).
///
/// A self-query is used so the expected answer is known independently of the
/// index, and the vectors are given distinct directions so the ordering is
/// strict — `vec![i; dim]` under cosine makes every point parallel to every
/// other, and any permutation of the ties would satisfy the assertion.
#[test]
#[cfg(feature = "persistence")]
fn analyze_preserves_the_points_a_query_returns() {
    assert_analyze_preserves_results(1_100);
}

/// The same collection below the reorder threshold, where no renumbering
/// happens — so a failure above can be attributed to the reorder rather than
/// to `ANALYZE`'s other work.
#[test]
#[cfg(feature = "persistence")]
fn analyze_below_the_reorder_threshold_preserves_results() {
    assert_analyze_preserves_results(900);
}

/// Builds an `n`-point collection, records a self-query's top-5, runs
/// `ANALYZE`, and requires the same five ids in the same order.
#[cfg(feature = "persistence")]
fn assert_analyze_preserves_results(n: u64) {
    let (_dir, db) = temp_database();
    db.create_collection("analyze_ids", 4, DistanceMetric::Euclidean)
        .expect("create collection");
    let coll = db
        .get_vector_collection("analyze_ids")
        .expect("collection exists");

    let points: Vec<Point> = (1..=n)
        .map(|i| Point {
            id: i,
            vector: vec![
                (i as f32 * 0.013).sin(),
                (i as f32 * 0.029).cos(),
                (i as f32 * 0.007).sin(),
                (i as f32 * 0.041).cos(),
            ],
            payload: None,
            sparse_vectors: None,
        })
        .collect();
    let query = points[499].vector.clone();
    coll.upsert(points).expect("upsert");

    let before = top_ids(&coll, &query);
    // The fixture is checked against the exhaustive scan, not the graph: on
    // this curve the graph loses point 500 itself in about one build in 400,
    // even at an ef wider than the collection, so not for want of beam
    // (#2259). What ANALYZE must preserve is the graph's answer, compared
    // below; whether that answer is exact is not this test's subject.
    let exact = coll
        .search_with_quality(&query, 1, crate::SearchQuality::Perfect)
        .expect("exhaustive search");
    assert_eq!(
        exact.first().map(|r| r.point.id),
        Some(500),
        "the fixture is only meaningful if the self-query finds itself first"
    );

    db.analyze_collection("analyze_ids").expect("analyze");

    assert_eq!(
        before,
        top_ids(&coll, &query),
        "ANALYZE changed the ids a query returns"
    );
}

/// The ids of a collection's top-5 for `query`.
#[cfg(feature = "persistence")]
fn top_ids(coll: &crate::VectorCollection, query: &[f32]) -> Vec<u64> {
    coll.search(query, 5)
        .expect("search")
        .iter()
        .map(|r| r.point.id)
        .collect()
}