xvc_pipeline/pipeline/deps/
sqlite_query.rs

1//! Implements dependencies on sqlite databases. [SqliteQueryDep] is a dependency that runs a query
2//! and checks whether the result of that query has changed. It doesn't run the query if the
3//! metadata of the database file hasn't changed.
4use crate::XvcDependency;
5use fallible_iterator::FallibleIterator;
6use rusqlite::{Connection, OpenFlags};
7use serde::{Deserialize, Serialize};
8
9use crate::Result;
10use xvc_core::types::diff::Diffable;
11use xvc_core::{ContentDigest, Diff, HashAlgorithm, XvcDigest, XvcMetadata, XvcPath, XvcRoot};
12
13use xvc_core::persist;
14
15/// When a step depends to a regex search in a text file
16#[derive(Debug, PartialOrd, Ord, Clone, Eq, PartialEq, Serialize, Deserialize)]
17pub struct SqliteQueryDep {
18    /// Path of the database file
19    pub path: XvcPath,
20    /// The query we run on the database
21    pub query: String,
22    /// The digest of the lines that match the regex
23    pub query_digest: Option<ContentDigest>,
24    /// The metadata of the database file
25    pub xvc_metadata: Option<XvcMetadata>,
26}
27
28persist!(SqliteQueryDep, "sqlite-query-dependency");
29
30impl From<SqliteQueryDep> for XvcDependency {
31    fn from(val: SqliteQueryDep) -> Self {
32        XvcDependency::SqliteQueryDigest(val)
33    }
34}
35
36impl SqliteQueryDep {
37    /// Create a new SqliteQueryDep with empty metadata and digest
38    pub fn new(path: XvcPath, query: String) -> Self {
39        Self {
40            path,
41            query,
42            query_digest: None,
43            xvc_metadata: None,
44        }
45    }
46    /// Update the metadata of the file
47    pub fn update_metadata(self, xvc_metadata: Option<XvcMetadata>) -> Self {
48        Self {
49            xvc_metadata,
50            ..self
51        }
52    }
53
54    /// Update the digest of the file by reading the file and collecting all lines that match the regex
55    pub fn update_digest(self, xvc_root: &XvcRoot, algorithm: HashAlgorithm) -> Result<Self> {
56        let path = self.path.to_absolute_path(xvc_root);
57        let flags = OpenFlags::SQLITE_OPEN_READ_ONLY;
58        let sqlite = Connection::open_with_flags(path, flags)?;
59        let mut prepared = sqlite.prepare(&self.query)?;
60        // TODO: Should we allow params in the queries?
61        let query_res = prepared.raw_query();
62        let query_lines = query_res
63            .map(|row| {
64                let mut i = 0;
65                // TODO: Add salting with the repo id here?
66                let mut els = String::new();
67                while let Ok(col) = row.get_ref(i) {
68                    match col.data_type() {
69                        rusqlite::types::Type::Text => {
70                            els.push_str(col.as_str()?);
71                        }
72                        rusqlite::types::Type::Integer => {
73                            els.push_str(col.as_i64()?.to_string().as_str());
74                        }
75                        rusqlite::types::Type::Real => {
76                            els.push_str(col.as_f64()?.to_string().as_str());
77                        }
78                        _ => {
79                            els.push_str(col.as_str()?);
80                        }
81                    }
82                    i += 1;
83                }
84                Ok(els)
85            })
86            .collect::<Vec<String>>()?
87            .join("\n");
88
89        let query_digest = Some(XvcDigest::from_content(&query_lines, algorithm).into());
90        Ok(Self {
91            query_digest,
92            ..self
93        })
94    }
95}
96
97impl Diffable for SqliteQueryDep {
98    type Item = Self;
99
100    /// ⚠️  Update the metadata with actual.update_metadata before calling this function
101    fn diff_superficial(record: &Self::Item, actual: &Self::Item) -> Diff<Self::Item> {
102        assert!(record.path == actual.path);
103
104        match (record.xvc_metadata, actual.xvc_metadata) {
105            (Some(rec_md), Some(act_md)) => {
106                if rec_md == act_md {
107                    Diff::Identical
108                } else {
109                    Diff::Different {
110                        record: record.clone(),
111                        actual: actual.clone(),
112                    }
113                }
114            }
115            (None, Some(_)) => Diff::RecordMissing {
116                actual: actual.clone(),
117            },
118            (Some(_), None) => Diff::ActualMissing {
119                record: record.clone(),
120            },
121            (None, None) => unreachable!("Either record or actual should have metadata"),
122        }
123    }
124
125    /// ⚠️  Update the metadata and lines with actual.update_digest before calling this function
126    fn diff_thorough(record: &Self::Item, actual: &Self::Item) -> Diff<Self::Item> {
127        assert!(record.path == actual.path);
128        if record.query_digest == actual.query_digest {
129            Diff::Identical
130        } else {
131            Diff::Different {
132                record: record.clone(),
133                actual: actual.clone(),
134            }
135        }
136    }
137}
xvc_pipeline/pipeline/deps/sqlite_query.rs

xvc_pipeline/pipeline/deps/
sqlite_query.rs
