pelicode_db/snapshot.rs
1//! Snapshots: copies of a database for checking only (specification ยง5.6).
2//!
3//! A snapshot keeps what `scan` reads: packages, files, functions, exact
4//! hashes, fingerprints, LSH buckets and the hash statistics. It drops the
5//! MinHash signatures, which only indexing and `export` use, and the
6//! bookkeeping of incremental statistics. Nothing can be indexed into it.
7
8use std::path::Path;
9
10use serde::Serialize;
11
12use crate::{Database, Error, Result};
13
14/// Key of `meta` marking a snapshot, with the time it was made.
15pub(crate) const SNAPSHOT_KEY: &str = "snapshot_created_at";
16/// Key of `meta` recording the fingerprint levels, since a snapshot has no
17/// MinHash signatures to read them from.
18pub(crate) const LEVELS_KEY: &str = "fingerprint_levels";
19
20/// What goes into a snapshot.
21#[derive(Debug, Clone, Default)]
22pub struct SnapshotOptions {
23 /// Only these ecosystems; all when empty.
24 pub ecosystems: Vec<String>,
25 /// Only the latest indexed version of each package, by publication time
26 /// (then crawl time).
27 pub latest_only: bool,
28}
29
30/// What [`Database::snapshot`] wrote.
31#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
32pub struct SnapshotSummary {
33 /// Packages in the snapshot.
34 pub packages: usize,
35 /// Package versions in the snapshot.
36 pub versions: usize,
37 /// Functions in the snapshot.
38 pub functions: usize,
39 /// Size of the snapshot file in bytes.
40 pub bytes: u64,
41 /// Ecosystems in the snapshot, by name.
42 pub ecosystems: Vec<String>,
43 /// The `meta` of the snapshot: versions, feature parameters, fingerprint
44 /// levels and the time it was made (see [`Database::meta`]).
45 pub meta: std::collections::BTreeMap<String, String>,
46}
47
48impl Database {
49 /// Writes a snapshot of this database to `out`, which must not exist.
50 /// This database is not changed.
51 ///
52 /// The snapshot holds the indexed versions selected by `options`, with
53 /// hash statistics recounted for them, in a single file (no write-ahead
54 /// log). It can be opened like any database, but indexing into it fails
55 /// with [`Error::Snapshot`].
56 ///
57 /// # Errors
58 ///
59 /// [`Error::Sqlite`] when `out` exists or cannot be written, or when this
60 /// database cannot be read.
61 ///
62 /// # Examples
63 ///
64 /// ```
65 /// use pelicode_core::{Config, Language, extract_bytes};
66 /// use pelicode_db::{Database, Error, PackageInfo, SnapshotOptions};
67 ///
68 /// let dir = tempfile::tempdir().unwrap();
69 /// let config = Config { min_tokens: 0, ..Config::default() };
70 /// let mut db = Database::open(dir.path().join("pelicode.db"), &config).unwrap();
71 /// let file = extract_bytes("a.py".as_ref(), Language::Python, b"def f(x):\n return x + 1\n", &config).unwrap();
72 /// let package = |ecosystem: &str, version: &str| PackageInfo {
73 /// ecosystem: ecosystem.into(),
74 /// name: "demo".into(),
75 /// version: version.into(),
76 /// ..PackageInfo::default()
77 /// };
78 /// db.index(&package("pypi", "1.0"), &[file.clone()]).unwrap();
79 /// db.index(&package("pypi", "1.1"), &[file.clone()]).unwrap();
80 /// db.index(&package("crates", "0.1.0"), &[file.clone()]).unwrap();
81 ///
82 /// let out = dir.path().join("snapshot.db");
83 /// let options = SnapshotOptions { ecosystems: vec!["pypi".into()], latest_only: true };
84 /// let summary = db.snapshot(&out, &options).unwrap();
85 /// assert_eq!((summary.packages, summary.versions), (1, 1));
86 /// assert_eq!(summary.ecosystems, ["pypi"]);
87 /// assert_eq!(summary.meta["fingerprint_levels"], "1");
88 ///
89 /// let mut snapshot = Database::open(&out, &config).unwrap();
90 /// assert!(snapshot.is_snapshot().unwrap());
91 /// assert!(matches!(snapshot.index(&package("pypi", "1.2"), &[file]), Err(Error::Snapshot)));
92 /// ```
93 pub fn snapshot(
94 &self,
95 out: impl AsRef<Path>,
96 options: &SnapshotOptions,
97 ) -> Result<SnapshotSummary> {
98 let out = out.as_ref();
99 let levels = self.fingerprint_levels()?;
100 self.conn
101 .execute("VACUUM INTO ?1", [out.to_string_lossy().as_ref()])?;
102
103 let mut copy = Database::open_unchecked(out)?;
104 copy.select(options)?;
105 copy.clean()?;
106 copy.rebuild_hash_stats()?;
107 copy.conn.execute_batch(
108 "DELETE FROM minhash;
109 DELETE FROM stats_pending_exact;
110 DELETE FROM stats_pending_fingerprint;",
111 )?;
112 copy.conn.execute(
113 "INSERT INTO meta (key, value) VALUES (?1, ?2), (?3, strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))
114 ON CONFLICT (key) DO UPDATE SET value = excluded.value",
115 [LEVELS_KEY, levels.as_str(), SNAPSHOT_KEY],
116 )?;
117 let stats = copy.stats()?;
118 let ecosystems = copy.ecosystems()?;
119 let meta = copy.meta()?;
120 copy.vacuum()?;
121 copy.conn.pragma_update(None, "journal_mode", "DELETE")?;
122 drop(copy);
123
124 Ok(SnapshotSummary {
125 packages: stats.packages,
126 versions: stats.versions,
127 functions: stats.functions,
128 bytes: std::fs::metadata(out).map(|m| m.len()).unwrap_or(0),
129 ecosystems,
130 meta,
131 })
132 }
133
134 /// Whether this database is a snapshot made by [`Database::snapshot`].
135 ///
136 /// # Errors
137 ///
138 /// [`Error::Sqlite`] when the database cannot be read.
139 ///
140 /// # Examples
141 ///
142 /// ```
143 /// use pelicode_core::Config;
144 /// use pelicode_db::Database;
145 ///
146 /// let dir = tempfile::tempdir().unwrap();
147 /// let db = Database::open(dir.path().join("pelicode.db"), &Config::default()).unwrap();
148 /// assert!(!db.is_snapshot().unwrap());
149 /// ```
150 pub fn is_snapshot(&self) -> Result<bool> {
151 Ok(self.meta()?.contains_key(SNAPSHOT_KEY))
152 }
153
154 pub(crate) fn refuse_snapshot(&self) -> Result<()> {
155 if self.is_snapshot()? {
156 return Err(Error::Snapshot);
157 }
158 Ok(())
159 }
160
161 /// The fingerprint levels, as a comma-separated list for `meta`.
162 fn fingerprint_levels(&self) -> Result<String> {
163 if let Some(levels) = self.meta()?.remove(LEVELS_KEY) {
164 return Ok(levels);
165 }
166 let mut stmt = self
167 .conn
168 .prepare("SELECT DISTINCT level FROM minhash ORDER BY level")?;
169 let levels: Vec<String> = stmt
170 .query_map([], |r| r.get::<_, i64>(0))?
171 .map(|l| l.map(|l| l.to_string()))
172 .collect::<rusqlite::Result<_>>()?;
173 Ok(levels.join(","))
174 }
175
176 /// Unlinks the versions `options` leave out, and removes them.
177 fn select(&mut self, options: &SnapshotOptions) -> Result<()> {
178 let tx = self.conn.transaction()?;
179 tx.execute_batch(
180 "DROP TABLE IF EXISTS temp.keep_version;
181 CREATE TEMP TABLE keep_version (id INTEGER PRIMARY KEY);
182 DROP TABLE IF EXISTS temp.keep_ecosystem;
183 CREATE TEMP TABLE keep_ecosystem (name TEXT PRIMARY KEY);",
184 )?;
185 for ecosystem in &options.ecosystems {
186 tx.execute(
187 "INSERT OR IGNORE INTO temp.keep_ecosystem (name) VALUES (?1)",
188 [ecosystem],
189 )?;
190 }
191 let latest = if options.latest_only {
192 "AND pv.id = (SELECT v.id FROM package_version v
193 WHERE v.package_id = pv.package_id AND v.status = 'indexed'
194 ORDER BY v.published_at IS NULL, v.published_at DESC,
195 v.crawled_at DESC, v.id DESC
196 LIMIT 1)"
197 } else {
198 ""
199 };
200 let sql = format!(
201 "INSERT INTO temp.keep_version (id)
202 SELECT pv.id FROM package_version pv
203 JOIN package p ON p.id = pv.package_id
204 JOIN ecosystem e ON e.id = p.ecosystem_id
205 WHERE pv.status = 'indexed'
206 AND (NOT EXISTS (SELECT 1 FROM temp.keep_ecosystem)
207 OR e.name IN (SELECT name FROM temp.keep_ecosystem))
208 {latest}"
209 );
210 tx.execute(&sql, [])?;
211 tx.execute_batch(
212 "DELETE FROM version_file
213 WHERE package_version_id NOT IN (SELECT id FROM temp.keep_version);
214 DELETE FROM package_version WHERE id NOT IN (SELECT id FROM temp.keep_version);
215 DELETE FROM package WHERE id NOT IN (SELECT package_id FROM package_version);
216 DELETE FROM ecosystem WHERE id NOT IN (SELECT ecosystem_id FROM package);
217 DROP TABLE temp.keep_version;
218 DROP TABLE temp.keep_ecosystem;",
219 )?;
220 tx.commit()?;
221 Ok(())
222 }
223}