moraine 0.6.3

A SlateDB-backed catalog for DuckLake
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

use std::sync::Arc;

use object_store::{ObjectStoreExt, memory::InMemory, path::Path};

use super::{AuxiliaryCache, Tier};
use crate::{CacheIdentity, DataStore, data_file::ParquetFile};

#[tokio::test]
async fn disk_recovery_keeps_object_namespaces_separate() {
    let directory =
        std::env::temp_dir().join(format!("moraine-cache-identity-{}", uuid::Uuid::new_v4()));
    let path = Path::from("same-file");
    let first = Arc::new(InMemory::new());
    let second = Arc::new(InMemory::new());
    let objects = [first, second];
    let namespaces = ["https://one.example/bucket", "https://two.example/bucket"];
    let values = [b"left", b"rght"];
    let cache = AuxiliaryCache::hybrid(1 << 20, &directory, 64 << 20)
        .await
        .unwrap();
    for index in 0..2 {
        objects[index]
            .put(&path, values[index].to_vec().into())
            .await
            .unwrap();
        let store = DataStore::with_cache_identity(
            objects[index].clone(),
            CacheIdentity::new(namespaces[index]),
        );
        let file = ParquetFile::new(store, path.clone(), 4, 0);
        assert_eq!(
            cache.range(&file, 0..4).await.unwrap().as_ref(),
            values[index]
        );
    }
    if let Tier::Hybrid(hybrid) = &cache.tier {
        hybrid.close().await.unwrap();
    }
    drop(cache);

    // With the source objects removed, successful reads must come from disk.
    for object in &objects {
        object.delete(&path).await.unwrap();
    }
    let cache = AuxiliaryCache::hybrid(1 << 20, &directory, 64 << 20)
        .await
        .unwrap();
    for index in [1, 0] {
        let store = DataStore::with_cache_identity(
            objects[index].clone(),
            CacheIdentity::new(namespaces[index]),
        );
        let file = ParquetFile::new(store, path.clone(), 4, 0);
        assert_eq!(
            cache.range(&file, 0..4).await.unwrap().as_ref(),
            values[index]
        );
    }
    let unnamed = ParquetFile::new(DataStore::new(objects[0].clone()), path, 4, 0);
    assert!(cache.range(&unnamed, 0..4).await.is_err());
    if let Tier::Hybrid(hybrid) = &cache.tier {
        hybrid.close().await.unwrap();
    }
    drop(cache);
    std::fs::remove_dir_all(directory).unwrap();
}