use crate::{Krate, Krates};
use anyhow::{Context, Error};
use log::{debug, info};
pub use rustsec::{advisory::Id, Database, Lockfile, Vulnerability};
use std::path::{Path, PathBuf};
use url::Url;
const DEFAULT_URL: &str = "https://github.com/RustSec/advisory-db";
#[derive(Copy, Clone)]
pub enum Fetch {
Allow,
AllowWithGitCli,
Disallow,
}
pub struct DbSet {
dbs: Vec<(Url, Database)>,
}
impl DbSet {
pub fn load(
root: Option<impl AsRef<Path>>,
mut urls: Vec<Url>,
fetch: Fetch,
) -> Result<Self, Error> {
let root_db_path = match root {
Some(root) => {
let user_root = root.as_ref();
if user_root.starts_with("~") {
if let Some(home) = home::home_dir() {
home.join(user_root.strip_prefix("~").unwrap())
} else {
log::warn!(
"unable to resolve path '{}', falling back to the default advisory path",
user_root.display()
);
home::cargo_home()
.context("failed to resolve CARGO_HOME")?
.join("advisory-dbs")
}
} else {
user_root.to_owned()
}
}
None => home::cargo_home()
.context("failed to resolve CARGO_HOME")?
.join("advisory-dbs"),
};
if urls.is_empty() {
info!(
"No advisory database configured, falling back to default '{}'",
DEFAULT_URL
);
urls.push(Url::parse(DEFAULT_URL).unwrap());
}
use rayon::prelude::*;
let mut dbs = Vec::with_capacity(urls.len());
urls.into_par_iter()
.map(|url| load_db(&url, root_db_path.clone(), fetch).map(|db| (url, db)))
.collect_into_vec(&mut dbs);
Ok(Self {
dbs: dbs.into_iter().collect::<Result<Vec<_>, _>>()?,
})
}
pub fn iter(&self) -> impl Iterator<Item = &(Url, Database)> {
self.dbs.iter()
}
pub fn has_advisory(&self, id: &Id) -> bool {
self.dbs.iter().any(|db| db.1.get(id).is_some())
}
}
pub(crate) fn url_to_local_dir(url: &str) -> Result<(String, String), Error> {
fn to_hex(num: u64) -> String {
const CHARS: &[u8] = b"0123456789abcdef";
let bytes = num.to_le_bytes();
let mut output = String::with_capacity(16);
for byte in bytes {
output.push(CHARS[(byte >> 4) as usize] as char);
output.push(CHARS[(byte & 0xf) as usize] as char);
}
output
}
#[allow(deprecated)]
fn hash_u64(url: &str) -> u64 {
use std::hash::{Hash, Hasher, SipHasher};
let mut hasher = SipHasher::new_with_keys(0, 0);
2u64.hash(&mut hasher);
url.hash(&mut hasher);
hasher.finish()
}
let (url, scheme_ind) = {
let scheme_ind = url
.find("://")
.with_context(|| format!("'{}' is not a valid url", url))?;
let scheme_str = &url[..scheme_ind];
if let Some(ind) = scheme_str.find('+') {
if &scheme_str[..ind] != "registry" {
anyhow::bail!("'{}' is not a valid registry url", url);
}
(&url[ind + 1..], scheme_ind - ind - 1)
} else {
(url, scheme_ind)
}
};
let host = match url[scheme_ind + 3..].find('/') {
Some(end) => &url[scheme_ind + 3..scheme_ind + 3 + end],
None => &url[scheme_ind + 3..],
};
let mut canonical = if host == "github.com" {
url.to_lowercase()
} else {
url.to_owned()
};
if let Some(hash) = canonical.rfind('#') {
canonical.truncate(hash);
}
if let Some(query) = canonical.rfind('?') {
canonical.truncate(query);
}
let ident = to_hex(hash_u64(&canonical));
if canonical.ends_with('/') {
canonical.pop();
}
if canonical.ends_with(".git") {
canonical.truncate(canonical.len() - 4);
}
Ok((format!("{}-{}", host, ident), canonical))
}
fn url_to_path(mut db_path: PathBuf, url: &Url) -> Result<PathBuf, Error> {
let (ident, _) = url_to_local_dir(url.as_str())?;
db_path.push(ident);
Ok(db_path)
}
fn load_db(db_url: &Url, root_db_path: PathBuf, fetch: Fetch) -> Result<Database, Error> {
let db_path = url_to_path(root_db_path, db_url)?;
match fetch {
Fetch::Allow => {
debug!("Fetching advisory database from '{db_url}'");
fetch_via_git(db_url, &db_path)
.with_context(|| format!("failed to fetch advisory database {db_url}"))?;
}
Fetch::AllowWithGitCli => {
debug!("Fetching advisory database with git cli from '{db_url}'");
fetch_via_cli(db_url.as_str(), &db_path)
.with_context(|| format!("failed to fetch advisory database {db_url} with cli"))?;
}
Fetch::Disallow => {
debug!("Opening advisory database at '{}'", db_path.display());
}
}
git2::Repository::open(&db_path).context("failed to open advisory database")?;
debug!("loading advisory database from {}", db_path.display());
let res = Database::open(&db_path).context("failed to load advisory database");
debug!(
"finished loading advisory database from {}",
db_path.display()
);
res
}
fn fetch_via_git(url: &Url, db_path: &Path) -> Result<(), Error> {
anyhow::ensure!(
url.scheme() == "https" || url.scheme() == "ssh",
"expected '{}' to be an `https` or `ssh` url",
url
);
{
let parent = db_path
.parent()
.with_context(|| format!("invalid directory: {}", db_path.display()))?;
if !parent.is_dir() {
std::fs::create_dir_all(parent)?;
}
}
if db_path.is_dir() && std::fs::read_dir(&db_path)?.next().is_none() {
std::fs::remove_dir(&db_path)?;
}
const LOCAL_REF: &str = "refs/heads/main";
const REMOTE_REF: &str = "refs/remotes/origin/main";
let git_config = git2::Config::new()?;
with_authentication(url.as_str(), &git_config, |f| {
let mut callbacks = git2::RemoteCallbacks::new();
callbacks.credentials(f);
let mut proxy_opts = git2::ProxyOptions::new();
proxy_opts.auto();
let mut fetch_opts = git2::FetchOptions::new();
fetch_opts.remote_callbacks(callbacks);
fetch_opts.proxy_options(proxy_opts);
if db_path.exists() {
let repo = git2::Repository::open(&db_path)?;
let refspec = format!("{LOCAL_REF}:{REMOTE_REF}");
let mut remote = repo.remote_anonymous(url.as_str())?;
remote.fetch(&[refspec.as_str()], Some(&mut fetch_opts), None)?;
let remote_main_ref = repo.find_reference(REMOTE_REF)?;
let remote_target = remote_main_ref.target().unwrap();
match repo.find_reference(LOCAL_REF) {
Ok(mut local_main_ref) => {
local_main_ref.set_target(
remote_target,
&format!("moving `main` to {}: {}", REMOTE_REF, &remote_target),
)?;
}
Err(e) if e.code() == git2::ErrorCode::NotFound => {
anyhow::bail!("unable to find reference '{}'", LOCAL_REF);
}
Err(e) => {
return Err(e.into());
}
};
} else {
git2::build::RepoBuilder::new()
.fetch_options(fetch_opts)
.clone(url.as_str(), db_path)?;
}
Ok(())
})?;
let repo = git2::Repository::open(&db_path).context("failed to open repository")?;
let head = repo.head()?;
let oid = head
.target()
.with_context(|| format!("no ref target for '{}'", db_path.display()))?;
let commit_object = repo.find_object(oid, Some(git2::ObjectType::Commit))?;
let commit = commit_object
.as_commit()
.context("HEAD OID was not a reference to a commit")?;
repo.reset(&commit_object, git2::ResetType::Hard, None)?;
let timestamp = time::OffsetDateTime::from_unix_timestamp(commit.time().seconds())
.context("commit timestamp is invalid")?;
const MINIMUM_FRESHNESS: time::Duration = time::Duration::seconds(90 * 24 * 60 * 60);
anyhow::ensure!(
timestamp
> time::OffsetDateTime::now_utc()
.checked_sub(MINIMUM_FRESHNESS)
.expect("this should never happen"),
"repository is stale (last commit: {})",
timestamp
);
Ok(())
}
fn fetch_via_cli(url: &str, db_path: &Path) -> Result<(), Error> {
use std::{fs, process::Command};
if let Some(parent) = db_path.parent() {
if !parent.is_dir() {
fs::create_dir_all(parent).with_context(|| {
format!(
"failed to create advisory database directory {}",
parent.display()
)
})?;
}
} else {
anyhow::bail!("invalid directory: {}", db_path.display());
}
fn capture(mut cmd: Command) -> Result<String, Error> {
cmd.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped());
let output = cmd
.spawn()
.context("failed to spawn git")?
.wait_with_output()
.context("failed to wait on git output")?;
if output.status.success() {
String::from_utf8(output.stdout)
.or_else(|_err| Ok("git command succeeded but gave non-utf8 output".to_owned()))
} else {
String::from_utf8(output.stderr)
.map_err(|_err| anyhow::anyhow!("git command failed and gave non-utf8 output"))
}
}
if db_path.exists() {
let mut cmd = Command::new("git");
cmd.arg("reset").arg("--hard").current_dir(db_path);
match capture(cmd) {
Ok(_reset) => log::debug!("reset {url}"),
Err(err) => log::error!("failed to reset {url}: {err}"),
}
let mut cmd = Command::new("git");
cmd.arg("pull").current_dir("/blah/nope");
capture(cmd).context("failed to pull latest changes")?;
log::debug!("pulled {url}");
} else {
let mut cmd = Command::new("git");
cmd.arg("clone").arg(url).arg(db_path);
capture(cmd).context("failed to clone")?;
log::debug!("cloned {url}");
}
Ok(())
}
pub fn with_authentication<T, F>(url: &str, cfg: &git2::Config, mut f: F) -> Result<T, Error>
where
F: FnMut(&mut git2::Credentials<'_>) -> Result<T, Error>,
{
let mut cred_helper = git2::CredentialHelper::new(url);
cred_helper.config(cfg);
let mut ssh_username_requested = false;
let mut cred_helper_bad = None;
let mut ssh_agent_attempts = Vec::new();
let mut any_attempts = false;
let mut tried_sshkey = false;
let mut res = f(&mut |url, username, allowed| {
any_attempts = true;
if allowed.contains(git2::CredentialType::USERNAME) {
debug_assert!(username.is_none());
ssh_username_requested = true;
return Err(git2::Error::from_str("gonna try usernames later"));
}
if allowed.contains(git2::CredentialType::SSH_KEY) && !tried_sshkey {
tried_sshkey = true;
let username = username.unwrap();
debug_assert!(!ssh_username_requested);
ssh_agent_attempts.push(username.to_string());
return git2::Cred::ssh_key_from_agent(username);
}
if allowed.contains(git2::CredentialType::USER_PASS_PLAINTEXT) && cred_helper_bad.is_none()
{
let r = git2::Cred::credential_helper(cfg, url, username);
cred_helper_bad = Some(r.is_err());
return r;
}
if allowed.contains(git2::CredentialType::DEFAULT) {
return git2::Cred::default();
}
Err(git2::Error::from_str("no authentication available"))
});
if ssh_username_requested {
debug_assert!(res.is_err());
let mut attempts = vec!["git".to_owned()];
if let Ok(s) = std::env::var("USER").or_else(|_| std::env::var("USERNAME")) {
attempts.push(s);
}
if let Some(s) = &cred_helper.username {
attempts.push(s.clone());
}
while let Some(s) = attempts.pop() {
let mut attempts = 0;
res = f(&mut |_url, username, allowed| {
if allowed.contains(git2::CredentialType::USERNAME) {
return git2::Cred::username(&s);
}
if allowed.contains(git2::CredentialType::SSH_KEY) {
debug_assert_eq!(Some(&s[..]), username);
attempts += 1;
if attempts == 1 {
ssh_agent_attempts.push(s.clone());
return git2::Cred::ssh_key_from_agent(&s);
}
}
Err(git2::Error::from_str("no authentication available"))
});
if attempts != 2 {
break;
}
}
}
if res.is_ok() || !any_attempts {
return res.map_err(From::from);
}
let res = res.map_err(|_e| {
let mut msg = "failed to authenticate when downloading repository".to_owned();
if !ssh_agent_attempts.is_empty() {
let names = ssh_agent_attempts
.iter()
.map(|s| format!("`{s}`"))
.collect::<Vec<_>>()
.join(", ");
use std::fmt::Write;
let _ = write!(
&mut msg,
"\nattempted ssh-agent authentication, but none of the usernames {names} succeeded",
);
}
if let Some(failed_cred_helper) = cred_helper_bad {
if failed_cred_helper {
msg.push_str(
"\nattempted to find username/password via \
git's `credential.helper` support, but failed",
);
} else {
msg.push_str(
"\nattempted to find username/password via \
`credential.helper`, but maybe the found \
credentials were incorrect",
);
}
}
anyhow::anyhow!(msg)
})?;
Ok(res)
}
pub fn load_lockfile(path: &krates::Utf8Path) -> Result<Lockfile, Error> {
let mut lockfile = Lockfile::load(path)?;
lockfile.metadata = Default::default();
Ok(lockfile)
}
pub struct PrunedLockfile(pub(crate) Lockfile);
impl PrunedLockfile {
pub fn prune(mut lf: Lockfile, krates: &Krates) -> Self {
lf.packages
.retain(|pkg| krate_for_pkg(krates, pkg).is_some());
Self(lf)
}
}
#[inline]
pub(crate) fn krate_for_pkg<'a>(
krates: &'a Krates,
pkg: &'a rustsec::package::Package,
) -> Option<(krates::NodeId, &'a Krate)> {
krates
.krates_by_name(pkg.name.as_str())
.find(|(_, krate)| {
pkg.version == krate.version
&& match (&pkg.source, &krate.source) {
(Some(psrc), Some(ksrc)) => psrc == ksrc,
(None, None) => true,
_ => false,
}
})
.map(|(ind, krate)| (ind, krate))
}
pub use rustsec::{Warning, WarningKind};
pub struct Report {
pub vulnerabilities: Vec<Vulnerability>,
pub notices: Vec<Warning>,
pub unmaintained: Vec<Warning>,
pub unsound: Vec<Warning>,
pub serialized_reports: Vec<serde_json::Value>,
}
impl Report {
pub fn generate(
advisory_dbs: &DbSet,
lockfile: &PrunedLockfile,
serialize_reports: bool,
) -> Self {
use rustsec::advisory::Informational;
let settings = rustsec::report::Settings {
target_arch: None,
target_os: None,
severity: None,
ignore: Vec::new(),
informational_warnings: vec![
Informational::Notice,
Informational::Unmaintained,
Informational::Unsound,
],
};
let mut vulnerabilities = Vec::new();
let mut notices = Vec::new();
let mut unmaintained = Vec::new();
let mut unsound = Vec::new();
let mut serialized_reports = Vec::with_capacity(if serialize_reports {
advisory_dbs.dbs.len()
} else {
0
});
for (url, db) in advisory_dbs.iter() {
let mut rep = rustsec::Report::generate(db, &lockfile.0, &settings);
if serialize_reports {
match serde_json::to_value(&rep) {
Ok(val) => serialized_reports.push(val),
Err(err) => {
log::error!("Failed to serialize report for database '{url}': {err}");
}
}
}
vulnerabilities.append(&mut rep.vulnerabilities.list);
for (kind, mut wi) in rep.warnings {
if wi.is_empty() {
continue;
}
match kind {
WarningKind::Notice => notices.append(&mut wi),
WarningKind::Unmaintained => unmaintained.append(&mut wi),
WarningKind::Unsound => unsound.append(&mut wi),
_ => unreachable!(),
}
}
}
Self {
vulnerabilities,
notices,
unmaintained,
unsound,
serialized_reports,
}
}
pub fn iter_warnings(&self) -> impl Iterator<Item = (WarningKind, &Warning)> {
self.notices
.iter()
.map(|wi| (WarningKind::Notice, wi))
.chain(
self.unmaintained
.iter()
.map(|wi| (WarningKind::Unmaintained, wi)),
)
.chain(self.unsound.iter().map(|wi| (WarningKind::Unsound, wi)))
}
}
#[cfg(test)]
mod test {
use super::url_to_path;
use url::Url;
#[test]
fn converts_url_to_path() {
let root_path = std::env::current_dir().unwrap();
{
let url = Url::parse("https://github.com/RustSec/advisory-db").unwrap();
assert_eq!(
url_to_path(root_path.clone(), &url).unwrap(),
root_path.join("github.com-2f857891b7f43c59")
);
}
{
let url = Url::parse("https://bare.com").unwrap();
assert_eq!(
url_to_path(root_path.clone(), &url).unwrap(),
root_path.join("bare.com-9c003d1ed306b28c")
);
}
{
let url = Url::parse("https://example.com/countries/việt nam").unwrap();
assert_eq!(
url_to_path(root_path.clone(), &url).unwrap(),
root_path.join("example.com-1c03f84825fb7438")
);
}
}
}