pelicode_db/
clean.rs

1//! Removing data that is no longer referenced.
2//!
3//! Indexing a package version again replaces its list of files; a file that no
4//! other version links to is then left behind with its functions and their
5//! features. They never appear in results, since they have no occurrence, but
6//! they take space.
7
8use serde::Serialize;
9
10use crate::{Database, Result};
11
12/// What [`Database::clean`] removed, or would remove.
13#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
14pub struct CleanSummary {
15    /// Source files no package version links to.
16    pub files: usize,
17    /// Functions of those files.
18    pub functions: usize,
19}
20
21const ORPHANS: &str = "
22    DROP TABLE IF EXISTS temp.orphan_file;
23    CREATE TEMP TABLE orphan_file AS
24        SELECT id FROM source_file
25        WHERE id NOT IN (SELECT source_file_id FROM version_file);
26    DROP TABLE IF EXISTS temp.orphan_function;
27    CREATE TEMP TABLE orphan_function AS
28        SELECT id FROM function WHERE source_file_id IN (SELECT id FROM orphan_file);
29";
30
31impl Database {
32    /// The files and functions [`Database::clean`] would remove, without removing them.
33    ///
34    /// # Errors
35    ///
36    /// [`Error`](crate::Error) when the database cannot be read.
37    ///
38    /// # Examples
39    ///
40    /// See [`Database::clean`].
41    pub fn unreferenced(&self) -> Result<CleanSummary> {
42        let count = |sql: &str| -> Result<usize> {
43            let n: i64 = self.conn.query_row(sql, [], |r| r.get(0))?;
44            Ok(n as usize)
45        };
46        Ok(CleanSummary {
47            files: count(
48                "SELECT COUNT(*) FROM source_file
49                 WHERE id NOT IN (SELECT source_file_id FROM version_file)",
50            )?,
51            functions: count(
52                "SELECT COUNT(*) FROM function WHERE source_file_id IN (
53                     SELECT id FROM source_file
54                     WHERE id NOT IN (SELECT source_file_id FROM version_file))",
55            )?,
56        })
57    }
58
59    /// Removes the source files no package version links to, with their
60    /// functions and features, in one transaction. The hash statistics are
61    /// not affected, since they count linked files only.
62    ///
63    /// # Errors
64    ///
65    /// [`Error`](crate::Error) when the database cannot be read or written;
66    /// nothing is removed then.
67    ///
68    /// # Examples
69    ///
70    /// ```
71    /// use pelicode_core::{Config, Language, extract_bytes};
72    /// use pelicode_db::{Database, PackageInfo};
73    ///
74    /// let dir = tempfile::tempdir().unwrap();
75    /// let config = Config::default();
76    /// let mut db = Database::open(dir.path().join("pelicode.db"), &config).unwrap();
77    /// let package = PackageInfo {
78    ///     ecosystem: "local".into(),
79    ///     name: "demo".into(),
80    ///     version: "1.0".into(),
81    ///     ..PackageInfo::default()
82    /// };
83    /// let file = |source: &[u8]| extract_bytes("a.py".as_ref(), Language::Python, source, &config).unwrap();
84    ///
85    /// db.index(&package, &[file(b"def f():\n    return 1\n")]).unwrap();
86    /// // Indexing the same version again leaves the first file unreferenced.
87    /// db.index(&package, &[file(b"def g():\n    return 2\n")]).unwrap();
88    /// assert_eq!(db.unreferenced().unwrap().files, 1);
89    ///
90    /// let removed = db.clean().unwrap();
91    /// assert_eq!((removed.files, removed.functions), (1, 1));
92    /// assert_eq!(db.unreferenced().unwrap().files, 0);
93    /// assert_eq!(db.stats().unwrap().functions, 1);
94    /// ```
95    pub fn clean(&mut self) -> Result<CleanSummary> {
96        let tx = self.conn.transaction()?;
97        tx.execute_batch(ORPHANS)?;
98        let count = |table: &str| -> Result<usize> {
99            let n: i64 = tx.query_row(&format!("SELECT COUNT(*) FROM temp.{table}"), [], |r| {
100                r.get(0)
101            })?;
102            Ok(n as usize)
103        };
104        let summary = CleanSummary {
105            files: count("orphan_file")?,
106            functions: count("orphan_function")?,
107        };
108        tx.execute_batch(
109            "DELETE FROM lsh_bucket  WHERE function_id IN (SELECT id FROM temp.orphan_function);
110             DELETE FROM minhash     WHERE function_id IN (SELECT id FROM temp.orphan_function);
111             DELETE FROM fingerprint WHERE function_id IN (SELECT id FROM temp.orphan_function);
112             DELETE FROM exact_hash  WHERE function_id IN (SELECT id FROM temp.orphan_function);
113             DELETE FROM function    WHERE id IN (SELECT id FROM temp.orphan_function);
114             DELETE FROM source_file WHERE id IN (SELECT id FROM temp.orphan_file);
115             DROP TABLE temp.orphan_function;
116             DROP TABLE temp.orphan_file;",
117        )?;
118        tx.commit()?;
119        Ok(summary)
120    }
121
122    /// Gives the space of removed rows back to the file system (SQLite `VACUUM`),
123    /// writing the write-ahead log into the database before and after.
124    ///
125    /// # Errors
126    ///
127    /// [`Error`](crate::Error) when another connection uses the database, or
128    /// when there is not enough disk space for the temporary copy `VACUUM` makes.
129    ///
130    /// # Examples
131    ///
132    /// ```
133    /// use pelicode_core::Config;
134    /// use pelicode_db::Database;
135    ///
136    /// let dir = tempfile::tempdir().unwrap();
137    /// let db = Database::open(dir.path().join("pelicode.db"), &Config::default()).unwrap();
138    /// db.vacuum().unwrap();
139    /// ```
140    pub fn vacuum(&self) -> Result<()> {
141        self.conn
142            // In WAL mode, VACUUM writes the new pages to the log too.
143            .execute_batch(
144                "PRAGMA wal_checkpoint(TRUNCATE); VACUUM; PRAGMA wal_checkpoint(TRUNCATE);",
145            )?;
146        Ok(())
147    }
148}
pelicode_db/clean.rs

pelicode_db/
clean.rs
