datafold 0.1.55

A personal database for data sovereignty with AI-powered ingestion
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

use super::DataFoldNode;
use crate::error::{FoldDbError, FoldDbResult};
use serde::Serialize;

#[derive(Debug, Clone, Serialize)]
pub struct TransformQueueInfo {
    pub queue: Vec<String>,
    pub length: usize,
    pub is_empty: bool,
}

impl DataFoldNode {
    /// Add a transform to the queue
    pub async fn add_transform_to_queue(&self, transform_id: &str) -> FoldDbResult<()> {
        let db = self.db.lock().await;
        let orchestrator = db.transform_orchestrator().ok_or_else(|| {
            FoldDbError::Config(
                "Transform orchestrator not available (requires Sled backend)".to_string(),
            )
        })?;
        orchestrator.add_transform(transform_id, "manual").await?;
        Ok(())
    }

    /// Get information about the transform queue
    pub async fn get_transform_queue_info(&self) -> FoldDbResult<TransformQueueInfo> {
        let db = self.db.lock().await;
        let orchestrator = db.transform_orchestrator().ok_or_else(|| {
            FoldDbError::Config(
                "Transform orchestrator not available (requires Sled backend)".to_string(),
            )
        })?;
        let queue = orchestrator.list_queued_transforms()?;
        let queue_length = queue.len();
        let is_empty = orchestrator.is_empty()?;
        Ok(TransformQueueInfo {
            queue,
            length: queue_length,
            is_empty,
        })
    }

    /// Get all backfill information
    pub async fn get_all_backfills(
        &self,
    ) -> FoldDbResult<Vec<crate::fold_db_core::infrastructure::backfill_tracker::BackfillInfo>>
    {
        let db = self.db.lock().await;
        Ok(db.get_all_backfills())
    }

    /// Get active backfills
    pub async fn get_active_backfills(
        &self,
    ) -> FoldDbResult<Vec<crate::fold_db_core::infrastructure::backfill_tracker::BackfillInfo>>
    {
        let db = self.db.lock().await;
        Ok(db.get_active_backfills())
    }

    /// Get specific backfill info by transform ID
    pub async fn get_backfill(
        &self,
        transform_id: &str,
    ) -> FoldDbResult<Option<crate::fold_db_core::infrastructure::backfill_tracker::BackfillInfo>>
    {
        let db = self.db.lock().await;
        Ok(db.get_backfill(transform_id))
    }

    /// Get specific backfill info by backfill hash
    pub async fn get_backfill_by_hash(
        &self,
        hash: &str,
    ) -> FoldDbResult<Option<crate::fold_db_core::infrastructure::backfill_tracker::BackfillInfo>>
    {
        let db = self.db.lock().await;
        Ok(db.get_backfill_tracker().get_backfill_by_hash(hash))
    }

    /// Get event statistics
    pub async fn get_event_statistics(
        &self,
    ) -> FoldDbResult<crate::fold_db_core::infrastructure::event_monitor::EventStatistics> {
        let db = self.db.lock().await;
        Ok(db.get_event_statistics())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::datafold_node::config::NodeConfig;

    use tempfile::tempdir;

    #[tokio::test]
    async fn queue_info_works() {
        let dir = tempdir().unwrap();
        let config = NodeConfig {
            database: crate::datafold_node::config::DatabaseConfig::Local {
                path: dir.path().to_path_buf(),
            },
            default_trust_distance: 1,
            network_listen_address: "/ip4/127.0.0.1/tcp/0".to_string(),
            security_config: Default::default(),
            schema_service_url: None,
        };
        let node = DataFoldNode::new(config).await.unwrap();
        let info = node.get_transform_queue_info().await.unwrap();
        assert!(info.is_empty);
        assert_eq!(info.length, 0);
    }
}