millipede-storage-memory 0.1.0

In-memory StorageClient implementation for the Millipede web crawler.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

//! Integration tests for the in-memory key-value store.

use bytes::Bytes;
use millipede_core::storage::{KeyValueStore, KeyValueStoreExt, ListKeysOptions};
use millipede_storage_memory::MemoryKeyValueStore;
use serde::{Deserialize, Serialize};
use std::sync::Arc;

#[tokio::test]
async fn bytes_and_typed_values_round_trip() {
    let store: Arc<dyn KeyValueStore> = Arc::new(MemoryKeyValueStore::new("test"));
    store
        .set_bytes("raw", Bytes::from_static(b"hello"), "text/plain")
        .await
        .unwrap();
    let entry = store.get_bytes("raw").await.unwrap().unwrap();
    assert_eq!(entry.value, Bytes::from_static(b"hello"));
    assert_eq!(entry.content_type, "text/plain");

    #[derive(Debug, Serialize, Deserialize, PartialEq)]
    struct Value {
        count: u32,
    }
    store.set("typed", &Value { count: 4 }).await.unwrap();
    assert_eq!(
        store.get::<Value>("typed").await.unwrap(),
        Some(Value { count: 4 })
    );
}

#[tokio::test]
async fn delete_is_idempotent() {
    let store = MemoryKeyValueStore::new("test");
    store
        .set_bytes("key", Bytes::from_static(b"value"), "text/plain")
        .await
        .unwrap();
    store.delete("key").await.unwrap();
    store.delete("key").await.unwrap();
    assert!(store.get_bytes("key").await.unwrap().is_none());
}

#[tokio::test]
async fn pagination_walks_sorted_keys_once() {
    let store = MemoryKeyValueStore::new("test");
    for key in ["e", "b", "d", "a", "c"] {
        store
            .set_bytes(key, Bytes::from_static(b"x"), "text/plain")
            .await
            .unwrap();
    }
    let mut start = None;
    let mut all = Vec::new();
    loop {
        let mut options = ListKeysOptions::default();
        options.limit = Some(2);
        options.exclusive_start_key = start;
        let page = store.list_keys(options).await.unwrap();
        all.extend(page.keys.iter().map(|key| key.key.clone()));
        assert!(page.keys.iter().all(|key| key.size == 1));
        if !page.is_truncated {
            assert!(page.next_exclusive_start_key.is_none());
            break;
        }
        assert_eq!(page.keys.len(), 2);
        start = page.next_exclusive_start_key;
    }
    assert_eq!(all, ["a", "b", "c", "d", "e"]);
}

#[tokio::test]
async fn zero_limit_does_not_report_an_unusable_truncated_page() {
    let store = MemoryKeyValueStore::new("test");
    store
        .set_bytes("key", Bytes::from_static(b"value"), "text/plain")
        .await
        .unwrap();
    let mut options = ListKeysOptions::default();
    options.limit = Some(0);

    let page = store.list_keys(options).await.unwrap();

    assert!(page.keys.is_empty());
    assert!(!page.is_truncated);
    assert!(page.next_exclusive_start_key.is_none());
}