hashtree-cli 0.2.139

Hashtree daemon and CLI with content-addressed storage and adaptive blob routing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

mod common;

use common::htree_bin;
use hashtree_core::store::Store;
use hashtree_lmdb::{compute_sha256, LmdbBlobStore};
use std::process::Command;

#[tokio::test]
async fn compact_handles_existing_large_map_size() {
    let tmp = tempfile::tempdir().expect("tempdir");
    let data_dir = tmp.path().join("data");
    let env_dir = data_dir.join("blobs");
    std::fs::create_dir_all(&env_dir).expect("env dir");

    let store = LmdbBlobStore::with_map_size(&env_dir, 64 * 1024 * 1024).expect("open lmdb store");
    let data = b"hello".to_vec();
    store
        .put(compute_sha256(&data), data)
        .await
        .expect("insert blob");
    drop(store);

    let before_bytes = std::fs::metadata(env_dir.join("data.mdb"))
        .expect("before data.mdb")
        .len();

    let output = Command::new(htree_bin())
        .arg("--data-dir")
        .arg(&data_dir)
        .arg("storage")
        .arg("compact")
        .arg("--env-dir")
        .arg("blobs")
        .output()
        .expect("run htree storage compact");

    assert!(
        output.status.success(),
        "htree storage compact failed.\nstdout:\n{}\nstderr:\n{}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr),
    );

    let after_bytes = std::fs::metadata(env_dir.join("data.mdb"))
        .expect("after data.mdb")
        .len();

    assert!(
        after_bytes < before_bytes,
        "expected compaction to shrink LMDB file"
    );
    assert!(!env_dir.join("data.mdb.compact").exists());
    assert!(!env_dir.join("data.mdb.bak").exists());
}

#[tokio::test]
async fn compact_uses_external_scratch_and_keeps_recovery_backup() {
    let tmp = tempfile::tempdir().expect("tempdir");
    let data_dir = tmp.path().join("data");
    let env_dir = data_dir.join("blobs");
    let scratch_dir = tmp.path().join("scratch");
    std::fs::create_dir_all(&env_dir).expect("env dir");

    let store = LmdbBlobStore::with_map_size(&env_dir, 64 * 1024 * 1024).expect("open store");
    let data = b"external scratch compaction".to_vec();
    store
        .put(compute_sha256(&data), data)
        .await
        .expect("insert blob");
    drop(store);
    let before_bytes = std::fs::metadata(env_dir.join("data.mdb"))
        .expect("before data.mdb")
        .len();

    let output = Command::new(htree_bin())
        .arg("--data-dir")
        .arg(&data_dir)
        .arg("storage")
        .arg("compact")
        .arg("--env-dir")
        .arg("blobs")
        .arg("--scratch-dir")
        .arg(&scratch_dir)
        .arg("--keep-backup")
        .output()
        .expect("run scratch compaction");
    assert!(
        output.status.success(),
        "scratch compaction failed.\nstdout:\n{}\nstderr:\n{}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr),
    );

    let after_bytes = std::fs::metadata(env_dir.join("data.mdb"))
        .expect("after data.mdb")
        .len();
    assert!(after_bytes < before_bytes);
    assert!(!env_dir.join("data.mdb.compact-recovery").exists());
    assert!(!env_dir.join("data.mdb.replacement").exists());

    let runs = std::fs::read_dir(&scratch_dir)
        .expect("scratch directory")
        .collect::<Result<Vec<_>, _>>()
        .expect("scratch runs");
    assert_eq!(runs.len(), 1);
    let backup = runs[0].path().join("data.mdb.bak");
    assert_eq!(
        std::fs::metadata(backup).expect("backup").len(),
        before_bytes
    );
    assert!(!runs[0].path().join("data.mdb.compact").exists());
}