pointcloud 0.5.5

An accessor layer for goko
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

//! Loaders for datasets. Just opens them up and returns a point cloud.

use std::path::{Path, PathBuf};

use crate::base_traits::*;
use crate::data_sources::*;
use crate::glued_data_cloud::*;
use crate::label_sources::*;
use crate::pc_errors::*;

mod yaml_loaders;
pub use yaml_loaders::*;
mod csv_loaders;
pub use csv_loaders::*;

/// Opens a set of memmaps of both data and labels
pub fn open_labeled_memmaps<M: Metric<[f32]>>(
    data_dim: usize,
    label_dim: usize,
    data_paths: &[PathBuf],
    labels_paths: &[PathBuf],
) -> PointCloudResult<HashGluedCloud<SimpleLabeledCloud<DataMemmap<M>, VecLabels>>> {
    if data_paths.len() != labels_paths.len() {
        panic!(
            "Mismatch of label and data paths Data: {:?}, Labels: {:?}",
            data_paths, labels_paths
        );
    }
    let collection: PointCloudResult<Vec<SimpleLabeledCloud<DataMemmap<M>, VecLabels>>> =
        data_paths
            .iter()
            .zip(labels_paths.iter())
            .map(|(dp, lp)| {
                let data = DataMemmap::<M>::new(data_dim, &dp)?;
                let labels = DataMemmap::<M>::new(label_dim, &lp)?.convert_to_labels();
                Ok(SimpleLabeledCloud::new(data, labels))
            })
            .collect();
    Ok(HashGluedCloud::new(collection?))
}

/// Opens a set of memmaps of just data
pub fn open_memmaps<M: Metric<[f32]>>(
    data_dim: usize,
    data_paths: &[PathBuf],
) -> PointCloudResult<HashGluedCloud<DataMemmap<M>>> {
    let collection: PointCloudResult<Vec<DataMemmap<M>>> = data_paths
        .iter()
        .map(|dp| DataMemmap::<M>::new(data_dim, &dp))
        .collect();
    Ok(HashGluedCloud::new(collection?))
}

/// Concatenates a glued data memmap to a single ram dataset
pub fn convert_glued_memmap_to_ram<M: Metric<[f32]>>(
    glued_cloud: HashGluedCloud<DataMemmap<M>>,
) -> DataRam<M> {
    glued_cloud
        .take_data_sources()
        .drain(0..)
        .map(|ds| ds.convert_to_ram())
        .reduce(|mut a, b| {
            a.merge(b);
            a
        })
        .unwrap()
}