javelin-tui 0.10.0

Display and work with Lance matrices
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

use crate::datasets::make_gaussian_cliques_multi;
use genegraph_storage::lance_storage_graph::LanceStorageGraph;
use genegraph_storage::metadata::GeneMetadata;
use genegraph_storage::traits::backend::StorageBackend;
use genegraph_storage::traits::metadata::Metadata;
use log::{debug, info};
use smartcore::linalg::basic::arrays::Array2;
use smartcore::linalg::basic::matrix::DenseMatrix;
use std::path::PathBuf;

/// Generate a toy dataset to showcase
pub async fn cmd_generate(n_items: usize, n_dims: usize, seed: u64) -> anyhow::Result<()> {
    // 1) Prepare storage directory and minimal metadata
    let name_id = "javelin_test";
    #[cfg(test)]
    fn prepare_dir(n_id: &str) -> PathBuf {
        let mut out_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
        out_dir.push("target");
        out_dir.push("debug");
        out_dir.push(n_id);
        out_dir
    }
    #[cfg(not(test))]
    fn prepare_dir(n_id: &str) -> PathBuf {
        let mut out_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
        out_dir.push(n_id);
        out_dir
    }
    let out_dir = prepare_dir(name_id);
    if out_dir.exists() {
        std::fs::remove_dir_all(&out_dir).unwrap();
    }

    let storage = LanceStorageGraph::new(
        out_dir.to_str().expect("non-UTF8 test path").to_string(),
        "javelin_test".to_string(),
    );

    // 2) Generate dense “full” dataset
    let (dense, sparse, vector) = make_gaussian_cliques_multi(n_items, 0.3, 5, n_dims, seed);
    let (nitems, nfeatures) = (dense.len(), dense[0].len());

    // Create metadata
    GeneMetadata::seed_metadata(&name_id, nitems, nfeatures, &storage)
        .await
        .unwrap();
    debug!("Saving metadata first to initialize storage directory");

    // add data to the storage
    let dense_matrix =
        DenseMatrix::<f64>::from_iterator(dense.iter().flatten().map(|x| *x), nitems, nfeatures, 0);
    storage
        .save_dense("raw_input", &dense_matrix, &storage.metadata_path())
        .await?;

    // the adjacency file and the norms file
    let mut md: GeneMetadata = storage.load_metadata().await.unwrap();
    let mock_info_adj = md.new_fileinfo(
        "adjacency",
        "sparse",
        (nitems, nitems),
        Some(sparse.nnz()),
        None,
    );
    let mock_info_norms = md.new_fileinfo("norms", "vector", (nitems, 1), None, None);

    md = md.add_file("adjacency", mock_info_adj);
    md = md.add_file("norms", mock_info_norms);

    storage
        .save_sparse("adjacency", &sparse, &storage.metadata_path())
        .await
        .unwrap();
    storage
        .save_vector("norms", &vector.as_slice(), &storage.metadata_path())
        .await
        .unwrap();

    // Fill any required fields on md.aspace_config, etc.
    storage.save_metadata(&md).await?;

    println!(
        "Generated example datasets in {:?}:
  - dense Lance:   {} rows × {} cols (raw_input)
  - sparse Lance:  (adjacency)
  - 1D vector Lance: (norms)",
        out_dir, nitems, nfeatures,
    );
    info!("Try now `javelin --filepath ./javelin_test`");

    Ok(())
}