lance-index 12.0.0

Lance indices implementation
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

//! Benchmark-only support for reopening a frozen two-file shuffle fixture.

use std::sync::Arc;

use lance_core::Result;
use lance_io::object_store::ObjectStore;
use object_store::path::Path;
use serde::{Deserialize, Serialize};

use super::shuffler::{ShuffleReader, TwoFileShuffleReader};
/// The path-independent metadata needed to reopen a two-file shuffle fixture.
///
/// This is public only so the external Criterion benchmark can serialize the
/// manifest on one revision and reopen the same data from another revision.
#[doc(hidden)]
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct TwoFileShuffleFixtureManifest {
    pub num_partitions: usize,
    pub num_flush_groups: u64,
    pub partition_counts: Vec<u64>,
    pub total_loss: f64,
}

/// Reopen a frozen local two-file shuffle fixture.
///
/// `output_dir` is deliberately separate from the manifest so a fixture can be
/// copied without embedding a machine-specific absolute path.
#[doc(hidden)]
pub async fn open_two_file_shuffle_fixture(
    output_dir: Path,
    manifest: &TwoFileShuffleFixtureManifest,
) -> Result<Box<dyn ShuffleReader>> {
    TwoFileShuffleReader::try_new(
        Arc::new(ObjectStore::local()),
        output_dir,
        manifest.num_partitions,
        manifest.num_flush_groups,
        manifest.partition_counts.clone(),
        manifest.total_loss,
    )
    .await
}