feltdb 0.4.1

Embedded durable state store: an append-only log with atomic multi-record transactions, version-conditional writes, and idempotent retries
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
/// Content addressable storage and acquisition module
use crate::{
    AcquisitionPolicy, ContentIdentity, ContentTransferError, FlowRef, PeerAdvertisement, PeerId,
};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};

/// A content addressable store that tracks where content lives
#[derive(Clone)]
pub struct ContentAddressableStore {
    /// Map from content hash to actual content
    content: Arc<Mutex<HashMap<String, Vec<u8>>>>,
    /// Map from content hash to which peers have it
    content_locations: Arc<Mutex<HashMap<String, Vec<PeerId>>>>,
}

impl ContentAddressableStore {
    /// Create a new content addressable store
    pub fn new() -> Self {
        Self {
            content: Arc::new(Mutex::new(HashMap::new())),
            content_locations: Arc::new(Mutex::new(HashMap::new())),
        }
    }

    /// Store content and track its hash
    pub fn store(&self, content: Vec<u8>, hash: String) -> Result<(), String> {
        let mut store = self.content.lock().map_err(|e| e.to_string())?;
        store.insert(hash, content);
        Ok(())
    }

    /// Retrieve content by hash
    pub fn get(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
        let store = self.content.lock().map_err(|e| e.to_string())?;
        Ok(store.get(hash).cloned())
    }

    /// Record that a peer has a piece of content
    pub fn record_location(&self, hash: String, peer: PeerId) -> Result<(), String> {
        let mut locations = self.content_locations.lock().map_err(|e| e.to_string())?;
        locations.entry(hash).or_insert_with(Vec::new).push(peer);
        Ok(())
    }

    /// Find peers that have a specific piece of content
    pub fn find_peers_for_content(&self, hash: &str) -> Result<Vec<PeerId>, String> {
        let locations = self.content_locations.lock().map_err(|e| e.to_string())?;
        Ok(locations.get(hash).cloned().unwrap_or_default())
    }

    /// Check if content exists locally
    pub fn has_content(&self, hash: &str) -> Result<bool, String> {
        let store = self.content.lock().map_err(|e| e.to_string())?;
        Ok(store.contains_key(hash))
    }
}

impl Default for ContentAddressableStore {
    fn default() -> Self {
        Self::new()
    }
}

/// Reference resolution with fallback to peer acquisition
pub struct DistributedReferenceResolver {
    /// Local content store
    _local_store: ContentAddressableStore,
}

impl DistributedReferenceResolver {
    /// Create a new distributed reference resolver
    pub fn new(local_store: ContentAddressableStore) -> Self {
        Self {
            _local_store: local_store,
        }
    }

    /// Try to resolve a reference, first locally then from peers
    pub fn resolve(
        &self,
        flow_ref: &FlowRef,
        available_peers: &[PeerAdvertisement],
    ) -> Result<Option<Vec<u8>>, String> {
        // Step 1: Try local resolution
        // Check if we have this content hash locally
        // For now, just indicate it wasn't found
        // In a real implementation, we'd query from local content store

        // Step 2: Look for peers that might have this
        for peer_ad in available_peers {
            if peer_ad.provides_namespace(&flow_ref.namespace) {
                // This peer might have the content
                // In a full implementation, we'd make a remote call here
                return Ok(None); // Placeholder
            }
        }

        Ok(None)
    }
}

/// Projection distribution - tracks where projections are hosted
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ProjectionDistribution {
    /// Namespace this projection is for
    pub namespace: String,
    /// Projection type (e.g., "search", "vector")
    pub projection_type: String,
    /// Peer hosting this projection
    pub hosted_by: PeerId,
    /// Last update timestamp
    pub updated_at_ms: u128,
}

impl ProjectionDistribution {
    /// Create a new projection distribution entry
    pub fn new(
        namespace: impl Into<String>,
        projection_type: impl Into<String>,
        hosted_by: PeerId,
    ) -> Self {
        Self {
            namespace: namespace.into(),
            projection_type: projection_type.into(),
            hosted_by,
            updated_at_ms: crate::now_ms(),
        }
    }
}

/// Capability location registry - tracks where capabilities are available
#[derive(Clone)]
pub struct CapabilityLocationRegistry {
    /// Map from capability name to list of peers providing it
    capability_locations: Arc<Mutex<HashMap<String, Vec<PeerId>>>>,
    /// Map from peer ID to capabilities it provides
    peer_capabilities: Arc<Mutex<HashMap<String, Vec<String>>>>,
}

impl CapabilityLocationRegistry {
    /// Create a new capability location registry
    pub fn new() -> Self {
        Self {
            capability_locations: Arc::new(Mutex::new(HashMap::new())),
            peer_capabilities: Arc::new(Mutex::new(HashMap::new())),
        }
    }

    /// Register that a peer provides a capability
    pub fn register_capability(&self, capability: String, peer: PeerId) -> Result<(), String> {
        let mut locations = self
            .capability_locations
            .lock()
            .map_err(|e| e.to_string())?;
        locations
            .entry(capability.clone())
            .or_insert_with(Vec::new)
            .push(peer.clone());

        let mut peer_caps = self.peer_capabilities.lock().map_err(|e| e.to_string())?;
        peer_caps
            .entry(peer.id.clone())
            .or_insert_with(Vec::new)
            .push(capability);

        Ok(())
    }

    /// Find peers that provide a capability
    pub fn peers_for_capability(&self, capability: &str) -> Result<Vec<PeerId>, String> {
        let locations = self
            .capability_locations
            .lock()
            .map_err(|e| e.to_string())?;
        Ok(locations.get(capability).cloned().unwrap_or_default())
    }

    /// Get all capabilities provided by a peer
    pub fn capabilities_for_peer(&self, peer_id: &PeerId) -> Result<Vec<String>, String> {
        let peer_caps = self.peer_capabilities.lock().map_err(|e| e.to_string())?;
        Ok(peer_caps.get(&peer_id.id).cloned().unwrap_or_default())
    }
}

impl Default for CapabilityLocationRegistry {
    fn default() -> Self {
        Self::new()
    }
}

/// Manager for acquiring content from distributed peers
#[derive(Clone)]
pub struct ContentAcquisitionManager {
    /// Local content store
    content_store: ContentAddressableStore,
    /// Peer registry for discovering providers
    peer_registry: Arc<Mutex<HashMap<String, PeerAdvertisement>>>,
}

impl ContentAcquisitionManager {
    /// Create a new content acquisition manager
    pub fn new(content_store: ContentAddressableStore) -> Self {
        Self {
            content_store,
            peer_registry: Arc::new(Mutex::new(HashMap::new())),
        }
    }

    /// Register a peer for content acquisition
    pub fn register_peer(&self, advertisement: PeerAdvertisement) -> Result<(), String> {
        let mut registry = self.peer_registry.lock().map_err(|e| e.to_string())?;
        registry.insert(advertisement.peer_id.id.clone(), advertisement);
        Ok(())
    }

    /// Resolve a reference using the acquisition policy
    pub fn resolve(
        &self,
        flow_ref: &FlowRef,
        policy: AcquisitionPolicy,
    ) -> Result<Option<Vec<u8>>, String> {
        // Step 1: Always try local first
        let local_key = format!("{}:{}", flow_ref.namespace, flow_ref.object_id);

        // Check local content store by namespace:id key
        // First, try to get it directly
        if let Ok(Some(content)) = self.content_store.get(&local_key) {
            return Ok(Some(content));
        }

        // Step 2: Handle acquisition policies
        match policy {
            AcquisitionPolicy::LocalOnly => {
                // Fail if not available locally
                Ok(None)
            }
            AcquisitionPolicy::PreferLocal
            | AcquisitionPolicy::RemoteAllowed
            | AcquisitionPolicy::Required => {
                // Try to find a peer that has this content
                // In a real implementation, this would be async and perform network I/O
                self.find_and_acquire_from_peer(flow_ref)
            }
        }
    }

    /// Find a peer that can provide content and attempt acquisition
    fn find_and_acquire_from_peer(&self, flow_ref: &FlowRef) -> Result<Option<Vec<u8>>, String> {
        let registry = self.peer_registry.lock().map_err(|e| e.to_string())?;

        // Find peers that provide this namespace
        let providers: Vec<_> = registry
            .values()
            .filter(|ad| ad.provides_namespace(&flow_ref.namespace))
            .collect();

        if providers.is_empty() {
            return Ok(None);
        }

        // For now, try the first available provider
        // In a real implementation, this would use load-aware selection
        if providers.first().is_some() {
            // Simulate content acquisition by checking what we already have
            // In reality, this would perform a remote RPC call
            Ok(None)
        } else {
            Ok(None)
        }
    }

    /// Verify content against its content identity
    pub fn verify_content(
        content: &[u8],
        expected_identity: &ContentIdentity,
    ) -> Result<(), ContentTransferError> {
        let computed_identity = ContentIdentity::from_bytes(content);

        if computed_identity.hash == expected_identity.hash {
            Ok(())
        } else {
            Err(ContentTransferError::HashMismatch {
                expected: expected_identity.hash.clone(),
                computed: computed_identity.hash,
            })
        }
    }

    /// Materialize acquired content into local storage
    pub fn materialize(
        &self,
        flow_ref: &FlowRef,
        content: Vec<u8>,
        content_identity: &ContentIdentity,
    ) -> Result<(), String> {
        // Verify content hash
        Self::verify_content(&content, content_identity).map_err(|e| e.to_string())?;

        // Store in content store
        let key = format!("{}:{}", flow_ref.namespace, flow_ref.object_id);
        self.content_store.store(content, key)?;

        Ok(())
    }

    /// Get content from local store
    pub fn get_local(&self, flow_ref: &FlowRef) -> Result<Option<Vec<u8>>, String> {
        let key = format!("{}:{}", flow_ref.namespace, flow_ref.object_id);
        self.content_store.get(&key)
    }

    /// Check if content exists locally
    pub fn has_local(&self, flow_ref: &FlowRef) -> Result<bool, String> {
        let key = format!("{}:{}", flow_ref.namespace, flow_ref.object_id);
        self.content_store.has_content(&key)
    }
}

impl Default for ContentAcquisitionManager {
    fn default() -> Self {
        Self::new(ContentAddressableStore::new())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_content_addressable_store() {
        let store = ContentAddressableStore::new();
        let content = b"hello world".to_vec();
        let hash = "abc123".to_string();

        store.store(content.clone(), hash.clone()).unwrap();
        let retrieved = store.get(&hash).unwrap();
        assert_eq!(retrieved, Some(content));
    }

    #[test]
    fn test_content_addressable_store_locations() {
        let store = ContentAddressableStore::new();
        let hash = "abc123".to_string();
        let peer_a = PeerId::new("peer-a");
        let peer_b = PeerId::new("peer-b");

        store.record_location(hash.clone(), peer_a).unwrap();
        store.record_location(hash.clone(), peer_b).unwrap();

        let peers = store.find_peers_for_content(&hash).unwrap();
        assert_eq!(peers.len(), 2);
    }

    #[test]
    fn test_projection_distribution() {
        let peer = PeerId::new("search-node");
        let proj = ProjectionDistribution::new("docs", "search", peer.clone());

        assert_eq!(proj.namespace, "docs");
        assert_eq!(proj.projection_type, "search");
        assert_eq!(proj.hosted_by.id, "search-node");
    }

    #[test]
    fn test_capability_location_registry() {
        let registry = CapabilityLocationRegistry::new();
        let peer_a = PeerId::new("peer-a");
        let peer_b = PeerId::new("peer-b");

        registry
            .register_capability("search:docs".to_string(), peer_a.clone())
            .unwrap();
        registry
            .register_capability("search:docs".to_string(), peer_b.clone())
            .unwrap();
        registry
            .register_capability("vector:embeddings".to_string(), peer_b.clone())
            .unwrap();

        let search_peers = registry.peers_for_capability("search:docs").unwrap();
        assert_eq!(search_peers.len(), 2);

        let vector_peers = registry.peers_for_capability("vector:embeddings").unwrap();
        assert_eq!(vector_peers.len(), 1);

        let peer_a_caps = registry.capabilities_for_peer(&peer_a).unwrap();
        assert_eq!(peer_a_caps.len(), 1);

        let peer_b_caps = registry.capabilities_for_peer(&peer_b).unwrap();
        assert_eq!(peer_b_caps.len(), 2);
    }

    #[test]
    fn test_content_acquisition_manager_local_only() {
        let store = ContentAddressableStore::new();
        let manager = ContentAcquisitionManager::new(store);

        let flow_ref = FlowRef::new("docs", "123");

        // Should not find content with LocalOnly policy
        let result = manager
            .resolve(&flow_ref, AcquisitionPolicy::LocalOnly)
            .unwrap();
        assert!(result.is_none());
    }

    #[test]
    fn test_content_acquisition_manager_materialize() {
        let store = ContentAddressableStore::new();
        let manager = ContentAcquisitionManager::new(store);

        let flow_ref = FlowRef::new("docs", "123");
        let content = b"test content".to_vec();
        let content_id = ContentIdentity::from_bytes(&content);

        // Materialize content
        manager
            .materialize(&flow_ref, content.clone(), &content_id)
            .unwrap();

        // Verify it's stored locally
        let retrieved = manager.get_local(&flow_ref).unwrap();
        assert_eq!(retrieved, Some(content));
    }

    #[test]
    fn test_content_acquisition_manager_has_local() {
        let store = ContentAddressableStore::new();
        let manager = ContentAcquisitionManager::new(store);

        let flow_ref = FlowRef::new("docs", "123");

        // Initially should not have content
        assert!(!manager.has_local(&flow_ref).unwrap());

        // Materialize content
        let content = b"test content".to_vec();
        let content_id = ContentIdentity::from_bytes(&content);
        manager
            .materialize(&flow_ref, content, &content_id)
            .unwrap();

        // Now should have it
        assert!(manager.has_local(&flow_ref).unwrap());
    }

    #[test]
    fn test_verify_content_success() {
        let content = b"test content".to_vec();
        let content_id = ContentIdentity::from_bytes(&content);

        let result = ContentAcquisitionManager::verify_content(&content, &content_id);
        assert!(result.is_ok());
    }

    #[test]
    fn test_verify_content_hash_mismatch() {
        let content = b"test content".to_vec();
        let wrong_id = ContentIdentity::new("wronghash");

        let result = ContentAcquisitionManager::verify_content(&content, &wrong_id);
        assert!(result.is_err());
    }
}