Struct Database 

pub struct Database { /* private fields */ }

A pelicode database: an SQLite file holding the features of functions and where they were found, but no source code.

Examples

use pelicode_core::{Config, Language, extract_bytes};
use pelicode_db::{Database, PackageInfo};

let dir = tempfile::tempdir().unwrap();
let config = Config::default();
let mut db = Database::open(dir.path().join("pelicode.db"), &config).unwrap();

let source = b"package demo\nfunc Add(a, b int) int { return a + b }\n";
let file = extract_bytes("add.go".as_ref(), Language::Go, source, &config).unwrap();
let package = PackageInfo {
    ecosystem: "local".into(),
    name: "demo".into(),
    version: "1.0".into(),
    ..PackageInfo::default()
};
db.index(&package, &[file]).unwrap();
assert_eq!(db.stats().unwrap().functions, 1);

Implementations

impl Database

pub fn unreferenced(&self) -> Result<CleanSummary>

The files and functions Database::clean would remove, without removing them.

Errors

Error when the database cannot be read.

Examples

See Database::clean.

pub fn clean(&mut self) -> Result<CleanSummary>

Removes the source files no package version links to, with their functions and features, in one transaction. The hash statistics are not affected, since they count linked files only.

Errors

Error when the database cannot be read or written; nothing is removed then.

Examples

use pelicode_core::{Config, Language, extract_bytes};
use pelicode_db::{Database, PackageInfo};

let dir = tempfile::tempdir().unwrap();
let config = Config::default();
let mut db = Database::open(dir.path().join("pelicode.db"), &config).unwrap();
let package = PackageInfo {
    ecosystem: "local".into(),
    name: "demo".into(),
    version: "1.0".into(),
    ..PackageInfo::default()
};
let file = |source: &[u8]| extract_bytes("a.py".as_ref(), Language::Python, source, &config).unwrap();

db.index(&package, &[file(b"def f():\n    return 1\n")]).unwrap();
// Indexing the same version again leaves the first file unreferenced.
db.index(&package, &[file(b"def g():\n    return 2\n")]).unwrap();
assert_eq!(db.unreferenced().unwrap().files, 1);

let removed = db.clean().unwrap();
assert_eq!((removed.files, removed.functions), (1, 1));
assert_eq!(db.unreferenced().unwrap().files, 0);
assert_eq!(db.stats().unwrap().functions, 1);

pub fn vacuum(&self) -> Result<()>

Gives the space of removed rows back to the file system (SQLite VACUUM), writing the write-ahead log into the database before and after.

Errors

Error when another connection uses the database, or when there is not enough disk space for the temporary copy VACUUM makes.

Examples

use pelicode_core::Config;
use pelicode_db::Database;

let dir = tempfile::tempdir().unwrap();
let db = Database::open(dir.path().join("pelicode.db"), &Config::default()).unwrap();
db.vacuum().unwrap();

impl Database

pub fn functions_by_exact_hash( &self, level: Level, hash: &Hash128, ) -> Result<Vec<i64>>

Functions whose exact hash at level is hash.

Errors

Error when the database cannot be read.

Examples

use pelicode_core::Level;

let ids = db.functions_by_exact_hash(Level::N1, &function.exact_hashes[&Level::N1]).unwrap();
assert_eq!(ids.len(), 1);

pub fn functions_by_lsh_bucket( &self, level: Level, band: usize, bucket: u64, ) -> Result<Vec<i64>>

Functions sharing LSH bucket bucket of band at level.

Errors

Error when the database cannot be read.

Examples

use pelicode_core::Level;

let buckets = &function.features[&Level::N1].lsh_buckets;
assert_eq!(db.functions_by_lsh_bucket(Level::N1, 0, buckets[0]).unwrap().len(), 1);

pub fn fingerprints( &self, function_id: i64, level: Level, ) -> Result<Vec<StoredFingerprint>>

Fingerprints of a stored function at level.

Errors

Error when the database cannot be read.

Examples

use pelicode_core::Level;

let id = db.functions_by_exact_hash(Level::N1, &function.exact_hashes[&Level::N1]).unwrap()[0];
let stored = db.fingerprints(id, Level::N1).unwrap();
assert_eq!(stored.len(), function.features[&Level::N1].fingerprints.len());
assert!(stored.iter().all(|f| (1..=5).contains(&f.start_line)));

pub fn function_info(&self, function_id: i64) -> Result<FunctionInfo>

A stored function, without its occurrences.

Errors

Error when no function has the identifier or the database cannot be read.

Examples

use pelicode_core::Level;

let id = db.functions_by_exact_hash(Level::N1, &function.exact_hashes[&Level::N1]).unwrap()[0];
let info = db.function_info(id).unwrap();
assert_eq!((info.name.as_str(), info.start_line, info.end_line), ("total", 1, 5));

pub fn occurrences(&self, function_id: i64) -> Result<Vec<Occurrence>>

Every place a stored function appears, earliest publication first.

Errors

Error when the database cannot be read.

Examples

use pelicode_core::Level;

let id = db.functions_by_exact_hash(Level::N1, &function.exact_hashes[&Level::N1]).unwrap()[0];
let occurrences = db.occurrences(id).unwrap();
assert_eq!(occurrences[0].package, "calc");
assert_eq!(occurrences[0].path, "src/calc.py");

impl Database

pub fn snapshot( &self, out: impl AsRef<Path>, options: &SnapshotOptions, ) -> Result<SnapshotSummary>

Writes a snapshot of this database to out, which must not exist. This database is not changed.

The snapshot holds the indexed versions selected by options, with hash statistics recounted for them, in a single file (no write-ahead log). It can be opened like any database, but indexing into it fails with Error::Snapshot.

Errors

Error::Sqlite when out exists or cannot be written, or when this database cannot be read.

Examples

use pelicode_core::{Config, Language, extract_bytes};
use pelicode_db::{Database, Error, PackageInfo, SnapshotOptions};

let dir = tempfile::tempdir().unwrap();
let config = Config { min_tokens: 0, ..Config::default() };
let mut db = Database::open(dir.path().join("pelicode.db"), &config).unwrap();
let file = extract_bytes("a.py".as_ref(), Language::Python, b"def f(x):\n    return x + 1\n", &config).unwrap();
let package = |ecosystem: &str, version: &str| PackageInfo {
    ecosystem: ecosystem.into(),
    name: "demo".into(),
    version: version.into(),
    ..PackageInfo::default()
};
db.index(&package("pypi", "1.0"), &[file.clone()]).unwrap();
db.index(&package("pypi", "1.1"), &[file.clone()]).unwrap();
db.index(&package("crates", "0.1.0"), &[file.clone()]).unwrap();

let out = dir.path().join("snapshot.db");
let options = SnapshotOptions { ecosystems: vec!["pypi".into()], latest_only: true };
let summary = db.snapshot(&out, &options).unwrap();
assert_eq!((summary.packages, summary.versions), (1, 1));
assert_eq!(summary.ecosystems, ["pypi"]);
assert_eq!(summary.meta["fingerprint_levels"], "1");

let mut snapshot = Database::open(&out, &config).unwrap();
assert!(snapshot.is_snapshot().unwrap());
assert!(matches!(snapshot.index(&package("pypi", "1.2"), &[file]), Err(Error::Snapshot)));

pub fn is_snapshot(&self) -> Result<bool>

Whether this database is a snapshot made by Database::snapshot.

Errors

Error::Sqlite when the database cannot be read.

Examples

use pelicode_core::Config;
use pelicode_db::Database;

let dir = tempfile::tempdir().unwrap();
let db = Database::open(dir.path().join("pelicode.db"), &Config::default()).unwrap();
assert!(!db.is_snapshot().unwrap());

impl Database

pub fn refresh_hash_stats(&mut self) -> Result<HashStatsInfo>

Brings the document frequencies up to date with the functions currently linked to packages: incrementally when possible (see HashStatsInfo::incremental), otherwise by recounting everything. Both give the same counts.

Errors

Error when the database cannot be read or written.

Examples

db.index(&package("calc"), &[file.clone()]).unwrap();
assert!(db.hash_stats_info().unwrap().stale);

// The first statistics are counted from scratch.
assert!(!db.hash_stats_info().unwrap().incremental);
let info = db.refresh_hash_stats().unwrap();
assert!(!info.stale);
assert_eq!(info.common_exact_hashes, 0);

// Later ones recount only the hashes of what was indexed since.
db.index(&package("copy"), &[file]).unwrap();
assert!(db.hash_stats_info().unwrap().incremental);
let info = db.refresh_hash_stats().unwrap();
assert_eq!(info.packages, 2);
assert!(info.common_exact_hashes > 0);

pub fn rebuild_hash_stats(&mut self) -> Result<HashStatsInfo>

Recounts the document frequencies of every hash, whatever was indexed since the last statistics.

Errors

Error when the database cannot be read or written.

Examples

use pelicode_core::Config;
use pelicode_db::Database;

let dir = tempfile::tempdir().unwrap();
let mut db = Database::open(dir.path().join("pelicode.db"), &Config::default()).unwrap();
let info = db.rebuild_hash_stats().unwrap();
assert!(info.updated_at.is_some());

pub fn hash_stats_info(&self) -> Result<HashStatsInfo>

State of the hash statistics, without rebuilding them.

Errors

Error when the database cannot be read.

Examples

use pelicode_core::Config;
use pelicode_db::Database;

let dir = tempfile::tempdir().unwrap();
let db = Database::open(dir.path().join("pelicode.db"), &Config::default()).unwrap();
let info = db.hash_stats_info().unwrap();
assert_eq!(info.updated_at, None);
assert_eq!(info.packages, 0);

pub fn package_count(&self) -> Result<usize>

Packages with at least one indexed file.

Errors

Error when the database cannot be read.

Examples

// The same function in two packages.
for name in ["calc", "copy"] {
    let package = PackageInfo {
        ecosystem: "pypi".into(),
        name: name.into(),
        version: "1.0".into(),
        ..PackageInfo::default()
    };
    db.index(&package, &[file.clone()]).unwrap();
}
assert_eq!(db.package_count().unwrap(), 2);

pub fn exact_package_count(&self, level: Level, hash: &Hash128) -> Result<usize>

Packages containing a function with this exact hash (1 when not common).

Counts as of the last Database::refresh_hash_stats.

Errors

Error when the database cannot be read.

Examples

// The same function in two packages.
for name in ["calc", "copy"] {
    let package = PackageInfo {
        ecosystem: "pypi".into(),
        name: name.into(),
        version: "1.0".into(),
        ..PackageInfo::default()
    };
    db.index(&package, &[file.clone()]).unwrap();
}
use pelicode_core::Level;

db.refresh_hash_stats().unwrap();
let hash = &file.functions[0].exact_hashes[&Level::N1];
assert_eq!(db.exact_package_count(Level::N1, hash).unwrap(), 2);

pub fn fingerprint_package_count( &self, level: Level, hash: u64, ) -> Result<usize>

Packages containing this fingerprint (1 when not common).

Counts as of the last Database::refresh_hash_stats.

Errors

Error when the database cannot be read.

Examples

// The same function in two packages.
for name in ["calc", "copy"] {
    let package = PackageInfo {
        ecosystem: "pypi".into(),
        name: name.into(),
        version: "1.0".into(),
        ..PackageInfo::default()
    };
    db.index(&package, &[file.clone()]).unwrap();
}
use pelicode_core::Level;

db.refresh_hash_stats().unwrap();
let fingerprint = file.functions[0].features[&Level::N1].fingerprints[0].hash;
assert_eq!(db.fingerprint_package_count(Level::N1, fingerprint).unwrap(), 2);
assert_eq!(db.fingerprint_package_count(Level::N1, 42).unwrap(), 1);

pub fn common_exact_hashes( &self, level: Level, min_tokens: usize, limit: usize, ) -> Result<Vec<CommonHash>>

The exact hashes at level of functions with at least min_tokens tokens, shared by the most packages.

Counts as of the last Database::refresh_hash_stats.

Errors

Error when the database cannot be read.

Examples

// The same function in two packages.
for name in ["calc", "copy"] {
    let package = PackageInfo {
        ecosystem: "pypi".into(),
        name: name.into(),
        version: "1.0".into(),
        ..PackageInfo::default()
    };
    db.index(&package, &[file.clone()]).unwrap();
}
use pelicode_core::Level;

db.refresh_hash_stats().unwrap();
let common = db.common_exact_hashes(Level::N1, 0, 10).unwrap();
assert_eq!(common[0].package_count, 2);
assert!(common[0].sample.as_deref().unwrap().contains("total"));

pub fn common_fingerprints( &self, level: Level, limit: usize, ) -> Result<Vec<CommonHash>>

The fingerprints at level shared by the most packages.

Counts as of the last Database::refresh_hash_stats.

Errors

Error when the database cannot be read.

Examples

// The same function in two packages.
for name in ["calc", "copy"] {
    let package = PackageInfo {
        ecosystem: "pypi".into(),
        name: name.into(),
        version: "1.0".into(),
        ..PackageInfo::default()
    };
    db.index(&package, &[file.clone()]).unwrap();
}
use pelicode_core::Level;

db.refresh_hash_stats().unwrap();
let common = db.common_fingerprints(Level::N1, 3).unwrap();
assert!(!common.is_empty() && common.len() <= 3);
assert!(common.iter().all(|c| c.package_count == 2));

impl Database

pub fn open(path: impl AsRef<Path>, config: &Config) -> Result<Database>

Opens (or creates) a database for features computed with config. Fails when the database was built with incompatible parameters.

Errors

Error::Incompatible when the database was built with other feature parameters or another table layout, and Error::Sqlite when the file cannot be opened.

Examples

use pelicode_core::Config;
use pelicode_db::{Database, Error};

let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("pelicode.db");
Database::open(&path, &Config::default()).unwrap();

let other = Config { kgram: 5, ..Config::default() };
assert!(matches!(Database::open(&path, &other), Err(Error::Incompatible { .. })));

pub fn open_unchecked(path: impl AsRef<Path>) -> Result<Database>

Opens a database without checking its parameters (for reading).

The table layout is still checked; use Database::config_from_meta to featurize code like the stored functions.

Errors

Error::Incompatible when the table layout differs, and Error::Sqlite when the file cannot be opened.

Examples

use pelicode_core::Config;
use pelicode_db::Database;

let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("pelicode.db");
Database::open(&path, &Config { kgram: 5, ..Config::default() }).unwrap();

let db = Database::open_unchecked(&path).unwrap();
assert_eq!(db.meta().unwrap()["kgram"], "5");

pub fn meta(&self) -> Result<BTreeMap<String, String>>

The parameters recorded in the database.

The keys are schema_version, normalizer_version, ident_mode, kgram, window, minhash_size and bands, and the bookkeeping of the hash statistics.

Errors

Error::Sqlite when the table cannot be read.

Examples

let db = Database::open(dir.path().join("pelicode.db"), &Config::default()).unwrap();
let meta = db.meta().unwrap();
assert_eq!(meta["kgram"], "10");
assert_eq!(meta["window"], "4");

pub fn config_from_meta(&self, base: &Config) -> Result<Config>

base with the parameters recorded in the database, so that queries are featurized exactly like the stored functions. Fingerprint levels become the levels that actually have fingerprints in the database.

Errors

Error::Incompatible when the database was built with other normalization rules, and Error::InvalidMeta when a recorded parameter cannot be parsed.

Examples

use pelicode_core::Config;
use pelicode_db::Database;

let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("pelicode.db");
Database::open(&path, &Config { kgram: 7, ..Config::default() }).unwrap();

let db = Database::open_unchecked(&path).unwrap();
let config = db.config_from_meta(&Config::default()).unwrap();
assert_eq!(config.kgram, 7);

pub fn index( &mut self, pkg: &PackageInfo, files: &[FileFeatures], ) -> Result<IndexSummary>

Stores files as the content of package version pkg, replacing what was previously recorded for that version. Files already in the database (same content hash) are linked without storing them again.

Errors

Error::Snapshot when this database is a snapshot, and Error::Sqlite when the files cannot be stored; nothing is stored then.

Examples

use pelicode_core::{Config, Language, extract_bytes};
use pelicode_db::{Database, PackageInfo};

let dir = tempfile::tempdir().unwrap();
let config = Config::default();
let mut db = Database::open(dir.path().join("pelicode.db"), &config).unwrap();
let file = extract_bytes("a.rb".as_ref(), Language::Ruby, b"def one\n  1\nend\n", &config).unwrap();
let package = |version: &str| PackageInfo {
    ecosystem: "rubygems".into(),
    name: "one".into(),
    version: version.into(),
    ..PackageInfo::default()
};

let first = db.index(&package("1.0"), &[file.clone()]).unwrap();
assert_eq!((first.files, first.new_files, first.functions), (1, 1, 1));

// The same file in another version is linked, not stored again.
let second = db.index(&package("1.1"), &[file]).unwrap();
assert_eq!(second.new_files, 0);

pub fn version_status( &self, ecosystem: &str, name: &str, version: &str, ) -> Result<Option<String>>

Status of a package version (indexed or failed), if recorded.

Errors

Error::Sqlite when the database cannot be read.

Examples

assert_eq!(db.version_status("pypi", "calc", "1.0").unwrap().as_deref(), Some("indexed"));
assert_eq!(db.version_status("pypi", "calc", "2.0").unwrap(), None);

pub fn record_failure(&mut self, pkg: &PackageInfo, error: &str) -> Result<()>

Records that fetching or indexing pkg failed, so that it can be retried later. Files already linked to the version are kept.

Errors

Error::Snapshot when this database is a snapshot, and Error::Sqlite when the failure cannot be stored.

Examples

use pelicode_core::Config;
use pelicode_db::{Database, PackageInfo};

let dir = tempfile::tempdir().unwrap();
let mut db = Database::open(dir.path().join("pelicode.db"), &Config::default()).unwrap();
let package = PackageInfo {
    ecosystem: "pypi".into(),
    name: "broken".into(),
    version: "0.1".into(),
    ..PackageInfo::default()
};
db.record_failure(&package, "digest mismatch").unwrap();
assert_eq!(db.version_status("pypi", "broken", "0.1").unwrap().as_deref(), Some("failed"));

pub fn stats(&self) -> Result<Stats>

Number of rows of the main tables.

Errors

Error::Sqlite when the database cannot be read.

Examples

let stats = db.stats().unwrap();
assert_eq!((stats.packages, stats.versions, stats.functions), (1, 1, 1));

pub fn package_names(&self, ecosystem: &str) -> Result<Vec<String>>

Names of the packages of ecosystem with at least one version, crawled or failed, sorted: the packages to check for new versions.

Errors

Error::Sqlite when the database cannot be read.

Examples

use pelicode_core::Config;
use pelicode_db::{Database, PackageInfo};

let dir = tempfile::tempdir().unwrap();
let mut db = Database::open(dir.path().join("pelicode.db"), &Config::default()).unwrap();
for name in ["six", "requests"] {
    let package = PackageInfo {
        ecosystem: "pypi".into(), name: name.into(), version: "1.0".into(), ..PackageInfo::default()
    };
    db.index(&package, &[]).unwrap();
}
assert_eq!(db.package_names("pypi").unwrap(), ["requests", "six"]);
assert!(db.package_names("crates").unwrap().is_empty());

pub fn updates_checked_at(&self, ecosystem: &str) -> Result<Option<String>>

When ecosystem was last checked for new versions (RFC 3339), if ever.

Errors

Error::Sqlite when the database cannot be read.

Examples

use pelicode_core::Config;
use pelicode_db::Database;

let dir = tempfile::tempdir().unwrap();
let db = Database::open(dir.path().join("pelicode.db"), &Config::default()).unwrap();
assert_eq!(db.updates_checked_at("pypi").unwrap(), None);
db.record_updates_check("pypi").unwrap();
assert!(db.updates_checked_at("pypi").unwrap().is_some());

pub fn record_updates_check(&self, ecosystem: &str) -> Result<()>

Records that ecosystem has been checked for new versions now.

Errors

Error::Snapshot when this database is a snapshot, and Error::Sqlite when it cannot be written.

Examples

See Database::updates_checked_at.

pub fn ecosystems(&self) -> Result<Vec<String>>

Names of the ecosystems that have packages, sorted.

Errors

Error::Sqlite when the database cannot be read.

Examples

use pelicode_core::{Config, Language, extract_bytes};
use pelicode_db::{Database, PackageInfo};

let dir = tempfile::tempdir().unwrap();
let config = Config::default();
let mut db = Database::open(dir.path().join("pelicode.db"), &config).unwrap();
let file = extract_bytes("a.py".as_ref(), Language::Python, b"def f():\n    return 1\n", &config).unwrap();
for ecosystem in ["pypi", "crates"] {
    let package = PackageInfo {
        ecosystem: ecosystem.into(), name: "demo".into(), version: "1".into(), ..PackageInfo::default()
    };
    db.index(&package, &[file.clone()]).unwrap();
}
assert_eq!(db.ecosystems().unwrap(), ["crates", "pypi"]);

pub fn functions(&self) -> Result<Vec<FunctionRow>>

Every occurrence of every stored function, ordered by package and location.

Errors

Error::Sqlite when the database cannot be read.

Examples

let rows = db.functions().unwrap();
assert_eq!(rows[0].name, "total");
assert_eq!(rows[0].path, "src/calc.py");
assert_eq!(rows[0].license.as_deref(), Some("MIT"));