#[cfg(all(feature = "use-jemalloc", feature = "system-alloc"))]
compile_error!("`use-jemalloc` and `system-alloc` are mutually exclusive");
#[cfg(all(feature = "use-jemalloc", feature = "use-mimalloc"))]
compile_error!("`use-jemalloc` and `use-mimalloc` are mutually exclusive");
#[cfg(all(feature = "system-alloc", not(target_os = "macos")))]
compile_error!("`system-alloc` is only supported on Darwin targets");
#[cfg(feature = "use-jemalloc")]
#[global_allocator]
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
#[cfg(all(
not(feature = "use-jemalloc"),
not(feature = "system-alloc"),
any(feature = "use-mimalloc", target_os = "linux", target_os = "windows")
))]
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
#[cfg(any(
feature = "system-alloc",
all(
not(feature = "use-jemalloc"),
not(feature = "system-alloc"),
not(any(feature = "use-mimalloc", target_os = "linux", target_os = "windows"))
)
))]
use std::alloc::System;
#[cfg(any(
feature = "system-alloc",
all(
not(feature = "use-jemalloc"),
not(feature = "system-alloc"),
not(any(feature = "use-mimalloc", target_os = "linux", target_os = "windows"))
)
))]
#[global_allocator]
static GLOBAL: System = System;
use std::{
io::{IsTerminal, Read, Write},
path::PathBuf,
sync::{Arc, Mutex},
time::Instant,
};
use anyhow::{Context, Result};
use console::Term;
use kingfisher::{
access_map, azure, bitbucket,
cli::{
self, CommandLineArgs, GlobalArgs,
commands::{
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
inputs::{ContentFilteringArgs, InputSpecifierArgs},
output::{OutputArgs, ReportOutputFormat},
rules::{
RuleCacheArgs, RuleCachePruneArgs, RuleSpecifierArgs, RulesCheckArgs, RulesCommand,
RulesCompileCacheArgs, RulesListArgs, RulesListOutputFormat,
},
},
global::Command,
},
direct_access_map, direct_revoke, direct_validate, findings_store,
findings_store::FindingsStore,
gitea, github, huggingface,
reporter::{DetailsReporter, ScanAuditContext, styles::Styles},
rule_loader::RuleLoader,
rules_database::{RuleCacheConfig, RuleCachePruneConfig, RulesDatabase, prune_rule_cache},
scanner::{load_and_record_rules, run_scan},
update::{check_for_update_async, rewrite_argv_for_reexec},
util::tokio_blocking_threads_limit,
validation::set_user_agent_suffix,
};
use serde_json::json;
use tempfile::TempDir;
use tokio::runtime::Builder;
use tracing::{error, info, warn};
use tracing_core::metadata::LevelFilter;
use tracing_subscriber::{
self, fmt, prelude::__tracing_subscriber_SubscriberExt, registry, util::SubscriberInitExt,
};
use url::Url;
use crate::cli::commands::{
azure::AzureRepoType,
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
gitea::GiteaRepoType,
gitlab::GitLabRepoType,
scan::{ListRepositoriesCommand, ScanOperation},
view,
};
fn main() -> anyhow::Result<()> {
raise_nproc_soft_limit();
const STACK_SIZE: usize = 32 * 1024 * 1024; let parser = std::thread::Builder::new()
.name("kingfisher-args".to_string())
.stack_size(STACK_SIZE)
.spawn(|| {
let (args, matches) = CommandLineArgs::parse_args_with_matches();
(Box::new(args), matches)
})
.context("Failed to spawn argument parser thread")?;
let (args, matches) = parser.join().unwrap_or_else(|e| std::panic::resume_unwind(e));
if let Command::Wizard(wizard) = &args.command {
#[cfg(feature = "gui")]
return kingfisher::wizard::run(
wizard.target.clone(),
wizard.report.clone(),
kingfisher::wizard::global_values(&matches),
);
#[cfg(not(feature = "gui"))]
{
let _ = wizard;
anyhow::bail!(
"This build does not include the native wizard. Build with `cargo build --release --features gui --bin kingfisher`, then run `kingfisher wizard` (alias: `kingfisher gui`)."
);
}
}
let builder =
std::thread::Builder::new().name("kingfisher-main".to_string()).stack_size(STACK_SIZE);
let handler = builder.spawn(move || run(*args, matches)).expect("failed to spawn main thread");
let result = handler.join().unwrap_or_else(|e| std::panic::resume_unwind(e));
if let Err(error) = &result
&& error.is::<kingfisher::scanner::NoScanInputsError>()
{
eprintln!("Error: {error:?}");
std::process::exit(3);
}
result
}
enum AsyncMainOutcome {
Done,
Reexec,
}
#[cfg(unix)]
fn raise_nproc_soft_limit() {
unsafe {
let mut rl = libc::rlimit { rlim_cur: 0, rlim_max: 0 };
if libc::getrlimit(libc::RLIMIT_NPROC, &mut rl) != 0 {
return;
}
if rl.rlim_cur < rl.rlim_max {
let new = libc::rlimit { rlim_cur: rl.rlim_max, rlim_max: rl.rlim_max };
let _ = libc::setrlimit(libc::RLIMIT_NPROC, &new);
}
}
}
#[cfg(not(unix))]
fn raise_nproc_soft_limit() {}
fn run(args: CommandLineArgs, matches: clap::ArgMatches) -> anyhow::Result<()> {
match rustls::crypto::aws_lc_rs::default_provider().install_default() {
Ok(()) => {}
Err(_already_installed) => {
warn!("rustls crypto provider was already installed; keeping existing provider");
}
}
set_user_agent_suffix(args.global_args.user_agent_suffix.clone());
let num_jobs = match &args.command {
Command::Scan(scan_args) => scan_args.scan_args.num_jobs,
Command::SelfUpdate => 1, Command::Rules(_) => std::thread::available_parallelism().map_or(1, |n| n.get()), Command::Validate(_) => 1, Command::Revoke(_) => 1, Command::BlastRadius(_) => 1,
Command::View(_) | Command::Wizard(_) => 1,
Command::Config(_) => 1,
};
let max_blocking = tokio_blocking_threads_limit(num_jobs);
let runtime = Builder::new_multi_thread()
.worker_threads(num_jobs)
.max_blocking_threads(max_blocking)
.thread_stack_size(8 * 1024 * 1024) .enable_all()
.build()
.context("Failed to create Tokio runtime")?;
let outcome = runtime.block_on(async_main(args, matches))?;
drop(runtime);
match outcome {
AsyncMainOutcome::Done => Ok(()),
AsyncMainOutcome::Reexec => {
if let Err(e) = reexec_with_new_binary() {
error!(
"Binary was updated but re-exec failed: {e}. The original command did not \
run. Re-run the command to use the new binary."
);
std::process::exit(1);
}
Ok(())
}
}
}
fn reexec_with_new_binary() -> std::io::Result<()> {
use std::process::Command;
let exe = std::env::current_exe()?;
let argv: Vec<std::ffi::OsString> = rewrite_argv_for_reexec(std::env::args_os());
if argv.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"cannot re-exec: process started with empty argv",
));
}
let _ = std::io::stdout().flush();
let _ = writeln!(std::io::stderr(), "Restarting with updated binary...");
let _ = std::io::stderr().flush();
let argv0 = argv[0].clone();
#[cfg(unix)]
{
use std::os::unix::process::CommandExt;
let err = Command::new(&exe).args(argv.iter().skip(1)).arg0(&argv0).exec();
Err(err)
}
#[cfg(windows)]
{
let _ = argv0;
let status = Command::new(&exe).args(argv.iter().skip(1)).status()?;
std::process::exit(status.code().unwrap_or(1));
}
#[cfg(not(any(unix, windows)))]
{
let _ = (exe, argv0);
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"re-exec is not supported on this platform",
))
}
}
fn setup_logging(global_args: &GlobalArgs) {
let (level, all_targets) = if global_args.quiet {
(LevelFilter::ERROR, false)
} else {
let level = match global_args.verbose {
0 => LevelFilter::INFO, 1 => LevelFilter::DEBUG, 2 => LevelFilter::TRACE, _ => LevelFilter::TRACE, };
let all_targets = global_args.verbose > 2; (level, all_targets)
};
let filter = if all_targets {
tracing_subscriber::filter::Targets::new().with_default(LevelFilter::TRACE)
} else {
tracing_subscriber::filter::Targets::new()
.with_default(LevelFilter::ERROR) .with_target("kingfisher", level) };
let fmt_layer = fmt::layer()
.with_writer(std::io::stderr) .with_target(true) .with_ansi(std::io::stderr().is_terminal()) .without_time(); registry()
.with(fmt_layer) .with(filter) .init();
}
fn load_project_config(
explicit: Option<&std::path::Path>,
) -> Result<Option<kingfisher::cli::config::KingfisherConfig>> {
let Some(p) = explicit else { return Ok(None) };
let bytes = std::fs::read(p).with_context(|| format!("read config {}", p.display()))?;
let yaml =
String::from_utf8(bytes).with_context(|| format!("config {} is not UTF-8", p.display()))?;
let cfg = kingfisher::cli::config::parse_str(&yaml)
.with_context(|| format!("parse config {}", p.display()))?;
info!("loaded config from {}", p.display());
Ok(Some(cfg))
}
fn apply_config(
scan_args: &mut cli::commands::scan::ScanArgs,
global_args: &mut GlobalArgs,
cfg: &kingfisher::cli::config::KingfisherConfig,
scan_matches: Option<&clap::ArgMatches>,
) {
use clap::parser::ValueSource;
fn config_wins(matches: Option<&clap::ArgMatches>, id: &str) -> bool {
matches!(matches.and_then(|m| m.value_source(id)), None | Some(ValueSource::DefaultValue))
}
fn api_url_config_wins(
matches: Option<&clap::ArgMatches>,
outer_id: &str,
subcommand: &str,
) -> bool {
if !config_wins(matches, outer_id) {
return false;
}
let sub = matches.and_then(|m| m.subcommand_matches(subcommand));
config_wins(sub, "api_url")
}
scan_args.skip_word.extend(cfg.filters.skip_words.iter().cloned());
scan_args.skip_regex.extend(cfg.filters.skip_regex.iter().cloned());
scan_args.content_filtering_args.exclude.extend(cfg.filters.exclude.iter().cloned());
if let Some(c) = cfg.scan.confidence
&& config_wins(scan_matches, "confidence")
{
scan_args.confidence = c.into();
}
if let Some(e) = cfg.scan.min_entropy
&& config_wins(scan_matches, "min_entropy")
{
scan_args.min_entropy = Some(e);
}
if let Some(v) = cfg.scan.no_validate
&& config_wins(scan_matches, "no_validate")
{
scan_args.no_validate = v;
}
if let Some(v) = cfg.scan.only_valid
&& config_wins(scan_matches, "only_valid")
&& config_wins(scan_matches, "validation_filter")
{
scan_args.only_valid = v;
}
if let Some(v) = cfg.scan.validation_filter
&& config_wins(scan_matches, "validation_filter")
&& config_wins(scan_matches, "only_valid")
{
scan_args.validation_filter = Some(v);
}
if let Some(v) = cfg.scan.redact
&& config_wins(scan_matches, "redact")
{
scan_args.redact = v;
}
if let Some(v) = cfg.scan.no_dedup
&& config_wins(scan_matches, "no_dedup")
{
scan_args.no_dedup = v;
}
if let Some(v) = cfg.scan.turbo
&& config_wins(scan_matches, "turbo")
{
scan_args.turbo = v;
}
if let Some(v) = cfg.scan.no_base64
&& config_wins(scan_matches, "no_base64")
{
scan_args.no_base64 = v;
}
if let Some(v) = cfg.scan.access_map
&& config_wins(scan_matches, "access_map")
{
scan_args.access_map = v;
}
if let Some(v) = cfg.scan.rule_stats
&& config_wins(scan_matches, "rule_stats")
{
scan_args.rule_stats = v;
}
if let Some(j) = cfg.scan.jobs
&& config_wins(scan_matches, "num_jobs")
{
scan_args.num_jobs = j;
}
if let Some(t) = cfg.scan.git_repo_timeout
&& config_wins(scan_matches, "git_repo_timeout")
{
scan_args.git_repo_timeout = t;
}
if !cfg.rules.enabled.is_empty() {
if config_wins(scan_matches, "rule") {
scan_args.rules.rule = cfg.rules.enabled.clone();
} else {
scan_args.rules.rule.extend(cfg.rules.enabled.iter().cloned());
}
}
scan_args.rules.rules_path.extend(cfg.rules.paths.iter().cloned());
scan_args.rules.exclude_rule.extend(cfg.rules.disabled.iter().cloned());
if let Some(v) = cfg.rules.load_builtins
&& config_wins(scan_matches, "load_builtins")
{
scan_args.rules.load_builtins = v;
}
if let Some(v) = cfg.rules.cache
&& config_wins(scan_matches, "rule_cache")
&& config_wins(scan_matches, "no_rule_cache")
{
scan_args.rule_cache.rule_cache = v;
scan_args.rule_cache.no_rule_cache = !v;
}
if let Some(path) = &cfg.rules.cache_dir
&& config_wins(scan_matches, "rule_cache_dir")
{
scan_args.rule_cache.rule_cache_dir = Some(path.clone());
}
if let Some(t) = cfg.validation.timeout
&& config_wins(scan_matches, "validation_timeout")
{
scan_args.validation_timeout = t;
}
if let Some(r) = cfg.validation.retries
&& config_wins(scan_matches, "validation_retries")
{
scan_args.validation_retries = r;
}
if let Some(rps) = cfg.validation.rps
&& config_wins(scan_matches, "validation_rps")
{
scan_args.validation_rps = Some(rps);
}
for (rule, rps) in &cfg.validation.rps_per_rule {
scan_args.validation_rps_rule.push(format!("{rule}={rps}"));
}
if let Some(v) = cfg.validation.full_response
&& config_wins(scan_matches, "full_validation_response")
{
scan_args.full_validation_response = v;
}
if let Some(n) = cfg.validation.max_response_length
&& config_wins(scan_matches, "max_validation_response_length")
{
scan_args.max_validation_response_length = n;
}
if let Some(mb) = cfg.filters.max_file_size_mb
&& config_wins(scan_matches, "max_file_size_mb")
{
scan_args.content_filtering_args.max_file_size_mb = mb;
}
if let Some(v) = cfg.filters.no_binary
&& config_wins(scan_matches, "no_binary")
{
scan_args.content_filtering_args.no_binary = v;
}
if let Some(v) = cfg.filters.no_extract_archives
&& config_wins(scan_matches, "no_extract_archives")
{
scan_args.content_filtering_args.no_extract_archives = v;
}
if let Some(d) = cfg.filters.extraction_depth
&& config_wins(scan_matches, "extraction_depth")
{
scan_args.content_filtering_args.extraction_depth = d;
}
if let Some(v) = cfg.filters.no_inline_ignore
&& config_wins(scan_matches, "no_inline_ignore")
{
scan_args.no_inline_ignore = v;
}
if let Some(v) = cfg.filters.no_ignore_if_contains
&& config_wins(scan_matches, "no_ignore_if_contains")
{
scan_args.no_ignore_if_contains = v;
}
scan_args.extra_ignore_comments.extend(cfg.filters.extra_ignore_comments.iter().cloned());
scan_args.skip_aws_account.extend(cfg.filters.skip_aws_accounts.iter().cloned());
if let Some(p) = &cfg.filters.skip_aws_account_file
&& config_wins(scan_matches, "skip_aws_account_file")
{
scan_args.skip_aws_account_file = Some(p.clone());
}
if let Some(f) = cfg.output.format
&& config_wins(scan_matches, "format")
{
scan_args.output_args.format = f.into();
}
if let Some(p) = &cfg.output.path
&& config_wins(scan_matches, "output")
{
scan_args.output_args.output = Some(p.clone());
}
if let Some(p) = &cfg.baseline.file
&& config_wins(scan_matches, "baseline_file")
{
scan_args.baseline_file = Some(p.clone());
}
if let Some(v) = cfg.baseline.manage
&& config_wins(scan_matches, "manage_baseline")
{
scan_args.manage_baseline = v;
}
if let Some(f) = cfg.alerts.defaults.format
&& config_wins(scan_matches, "alert_format")
{
scan_args.alert_format = Some(f);
}
if let Some(o) = cfg.alerts.defaults.on
&& config_wins(scan_matches, "alert_on")
{
scan_args.alert_on = o;
}
if let Some(c) = cfg.alerts.defaults.min_confidence
&& config_wins(scan_matches, "alert_min_confidence")
{
scan_args.alert_min_confidence = c.into();
}
if let Some(v) = cfg.alerts.defaults.include_secret
&& config_wins(scan_matches, "alert_include_secret")
{
scan_args.alert_include_secret = v;
}
if let Some(u) = &cfg.alerts.defaults.report_url
&& config_wins(scan_matches, "alert_report_url")
{
scan_args.alert_report_url = Some(u.clone());
}
if let Some(d) = cfg.alerts.defaults.detail
&& config_wins(scan_matches, "alert_detail")
{
scan_args.alert_detail = d;
}
if let Some(f) = cfg.alerts.defaults.finding_filter
&& config_wins(scan_matches, "alert_finding_filter")
{
scan_args.alert_finding_filter = f;
}
if let Some(v) = cfg.alerts.defaults.prevent_empty
&& config_wins(scan_matches, "alert_prevent_empty")
{
scan_args.alert_prevent_empty = v;
}
for w in &cfg.alerts.webhooks {
scan_args.alert_webhook.push(w.url.clone());
scan_args.config_webhook_overrides.push(
kingfisher::cli::commands::scan::ConfigWebhookOverride {
format: w.format,
on: w.on,
min_confidence: w.min_confidence.map(Into::into),
include_secret: w.include_secret,
report_url: w.report_url.clone(),
detail: w.detail,
finding_filter: w.finding_filter,
prevent_empty: w.prevent_empty,
},
);
}
if let Some(m) = cfg.global.tls_mode
&& config_wins(scan_matches, "tls_mode")
{
global_args.tls_mode = m.into();
}
if let Some(v) = cfg.global.allow_internal_ips
&& config_wins(scan_matches, "allow_internal_ips")
{
global_args.allow_internal_ips = v;
}
if let Some(v) = cfg.global.no_update_check
&& config_wins(scan_matches, "no_update_check")
{
global_args.no_update_check = v;
}
if let Some(s) = &cfg.global.user_agent_suffix
&& config_wins(scan_matches, "user_agent_suffix")
{
let trimmed = s.trim();
if !trimmed.is_empty() {
global_args.user_agent_suffix = Some(trimmed.to_string());
}
}
global_args.endpoint.extend(cfg.global.endpoints.iter().cloned());
if let Some(p) = &cfg.global.endpoint_config
&& config_wins(scan_matches, "endpoint_config")
{
global_args.endpoint_config = Some(p.clone());
}
if let Some(p) = &cfg.git.clone_dir
&& config_wins(scan_matches, "git_clone_dir")
{
scan_args.input_specifier_args.git_clone_dir = Some(p.clone());
}
if let Some(v) = cfg.git.keep_clones
&& config_wins(scan_matches, "keep_clones")
{
scan_args.input_specifier_args.keep_clones = v;
}
if let Some(n) = cfg.git.repo_clone_limit
&& config_wins(scan_matches, "repo_clone_limit")
{
scan_args.input_specifier_args.repo_clone_limit = Some(n);
}
if let Some(v) = cfg.git.include_contributors
&& config_wins(scan_matches, "include_contributors")
{
scan_args.input_specifier_args.include_contributors = v;
}
if let Some(u) = &cfg.git.github_api_url
&& api_url_config_wins(scan_matches, "github_api_url", "github")
&& let Ok(parsed) = url::Url::parse(u)
{
scan_args.input_specifier_args.github_api_url = parsed;
}
if let Some(u) = &cfg.git.gitlab_api_url
&& api_url_config_wins(scan_matches, "gitlab_api_url", "gitlab")
&& let Ok(parsed) = url::Url::parse(u)
{
scan_args.input_specifier_args.gitlab_api_url = parsed;
}
}
fn run_config_command(
config_args: kingfisher::cli::commands::config_command::ConfigArgs,
global_args: &GlobalArgs,
top_matches: &clap::ArgMatches,
) -> Result<()> {
use kingfisher::cli::commands::config_command::ConfigSubcommand;
match config_args.command {
ConfigSubcommand::Init(init_args) => {
let init_matches = top_matches
.subcommand_matches("config")
.and_then(|m| m.subcommand_matches("init"))
.ok_or_else(|| anyhow::anyhow!("internal: missing `config init` matches"))?;
let yaml = build_config_yaml(&init_args.scan_args, global_args, init_matches)?;
match init_args.out.as_deref() {
Some(path) => {
if !init_args.force && path.exists() {
anyhow::bail!(
"{} already exists. Pass --force to overwrite.",
path.display()
);
}
std::fs::write(path, &yaml)
.with_context(|| format!("write {}", path.display()))?;
info!("wrote {}", path.display());
}
None => {
let mut stdout = std::io::stdout().lock();
stdout.write_all(yaml.as_bytes())?;
}
}
}
}
Ok(())
}
fn build_config_yaml(
scan_args: &cli::commands::scan::ScanArgs,
global_args: &GlobalArgs,
sub_matches: &clap::ArgMatches,
) -> Result<String> {
use clap::parser::ValueSource;
use kingfisher::cli::config::{
AlertsConfig, AlertsDefaultsConfig, BaselineConfig, FiltersConfig, GitConfig, GlobalConfig,
KingfisherConfig, OutputConfig, RulesConfig, ScanConfig, ValidationConfig, WebhookConfig,
};
use std::collections::BTreeMap;
fn user_set(matches: &clap::ArgMatches, id: &str) -> bool {
matches!(
matches.value_source(id),
Some(ValueSource::CommandLine | ValueSource::EnvVariable)
)
}
let mut cfg = KingfisherConfig::default();
let mut scan = ScanConfig::default();
if user_set(sub_matches, "confidence") {
scan.confidence = Some(scan_args.confidence.into());
}
if user_set(sub_matches, "min_entropy")
&& let Some(e) = scan_args.min_entropy
{
scan.min_entropy = Some(e);
}
if user_set(sub_matches, "no_validate") {
scan.no_validate = Some(scan_args.no_validate);
}
if user_set(sub_matches, "only_valid") {
scan.only_valid = Some(scan_args.only_valid);
}
if user_set(sub_matches, "validation_filter") {
scan.validation_filter = scan_args.validation_filter;
}
if user_set(sub_matches, "redact") {
scan.redact = Some(scan_args.redact);
}
if user_set(sub_matches, "no_dedup") {
scan.no_dedup = Some(scan_args.no_dedup);
}
if user_set(sub_matches, "turbo") {
scan.turbo = Some(scan_args.turbo);
}
if user_set(sub_matches, "no_base64") {
scan.no_base64 = Some(scan_args.no_base64);
}
if user_set(sub_matches, "access_map") {
scan.access_map = Some(scan_args.access_map);
}
if user_set(sub_matches, "rule_stats") {
scan.rule_stats = Some(scan_args.rule_stats);
}
if user_set(sub_matches, "num_jobs") {
scan.jobs = Some(scan_args.num_jobs);
}
if user_set(sub_matches, "git_repo_timeout") {
scan.git_repo_timeout = Some(scan_args.git_repo_timeout);
}
cfg.scan = scan;
let mut rules = RulesConfig::default();
if user_set(sub_matches, "rule") {
rules.enabled = scan_args.rules.rule.clone();
}
if user_set(sub_matches, "exclude_rule") {
rules.disabled = scan_args.rules.exclude_rule.clone();
}
if !scan_args.rules.rules_path.is_empty() {
rules.paths = scan_args.rules.rules_path.clone();
}
if user_set(sub_matches, "load_builtins") {
rules.load_builtins = Some(scan_args.rules.load_builtins);
}
if user_set(sub_matches, "rule_cache") {
rules.cache = Some(true);
}
if user_set(sub_matches, "no_rule_cache") {
rules.cache = Some(false);
}
if user_set(sub_matches, "rule_cache_dir") {
rules.cache_dir = scan_args.rule_cache.rule_cache_dir.clone();
}
cfg.rules = rules;
let mut validation = ValidationConfig::default();
if user_set(sub_matches, "validation_timeout") {
validation.timeout = Some(scan_args.validation_timeout);
}
if user_set(sub_matches, "validation_retries") {
validation.retries = Some(scan_args.validation_retries);
}
if user_set(sub_matches, "validation_rps")
&& let Some(rps) = scan_args.validation_rps
{
validation.rps = Some(rps);
}
if !scan_args.validation_rps_rule.is_empty() {
let mut map = BTreeMap::new();
for entry in &scan_args.validation_rps_rule {
let (rule, rps) = entry
.split_once('=')
.ok_or_else(|| anyhow::anyhow!("invalid --validation-rps-rule entry: {entry:?}"))?;
let rps: f64 = rps.parse().with_context(|| format!("invalid RPS in {entry:?}"))?;
map.insert(rule.trim().to_string(), rps);
}
validation.rps_per_rule = map;
}
if user_set(sub_matches, "full_validation_response") {
validation.full_response = Some(scan_args.full_validation_response);
}
if user_set(sub_matches, "max_validation_response_length") {
validation.max_response_length = Some(scan_args.max_validation_response_length);
}
cfg.validation = validation;
let mut filters = FiltersConfig::default();
if !scan_args.skip_word.is_empty() {
filters.skip_words = scan_args.skip_word.clone();
}
if !scan_args.skip_regex.is_empty() {
filters.skip_regex = scan_args.skip_regex.clone();
}
if !scan_args.content_filtering_args.exclude.is_empty() {
filters.exclude = scan_args.content_filtering_args.exclude.clone();
}
if user_set(sub_matches, "max_file_size_mb") {
filters.max_file_size_mb = Some(scan_args.content_filtering_args.max_file_size_mb);
}
if user_set(sub_matches, "no_binary") {
filters.no_binary = Some(scan_args.content_filtering_args.no_binary);
}
if user_set(sub_matches, "no_extract_archives") {
filters.no_extract_archives = Some(scan_args.content_filtering_args.no_extract_archives);
}
if user_set(sub_matches, "extraction_depth") {
filters.extraction_depth = Some(scan_args.content_filtering_args.extraction_depth);
}
if user_set(sub_matches, "no_inline_ignore") {
filters.no_inline_ignore = Some(scan_args.no_inline_ignore);
}
if user_set(sub_matches, "no_ignore_if_contains") {
filters.no_ignore_if_contains = Some(scan_args.no_ignore_if_contains);
}
if !scan_args.extra_ignore_comments.is_empty() {
filters.extra_ignore_comments = scan_args.extra_ignore_comments.clone();
}
if !scan_args.skip_aws_account.is_empty() {
filters.skip_aws_accounts = scan_args.skip_aws_account.clone();
}
if user_set(sub_matches, "skip_aws_account_file")
&& let Some(p) = &scan_args.skip_aws_account_file
{
filters.skip_aws_account_file = Some(p.clone());
}
cfg.filters = filters;
let mut output = OutputConfig::default();
if user_set(sub_matches, "format") {
output.format = Some(scan_args.output_args.format.into());
}
if user_set(sub_matches, "output")
&& let Some(p) = &scan_args.output_args.output
{
output.path = Some(p.clone());
}
cfg.output = output;
let mut baseline = BaselineConfig::default();
if user_set(sub_matches, "baseline_file")
&& let Some(p) = &scan_args.baseline_file
{
baseline.file = Some(p.clone());
}
if user_set(sub_matches, "manage_baseline") {
baseline.manage = Some(scan_args.manage_baseline);
}
cfg.baseline = baseline;
let mut alerts = AlertsConfig::default();
let mut defaults = AlertsDefaultsConfig::default();
if user_set(sub_matches, "alert_format") {
defaults.format = scan_args.alert_format;
}
if user_set(sub_matches, "alert_on") {
defaults.on = Some(scan_args.alert_on);
}
if user_set(sub_matches, "alert_min_confidence") {
defaults.min_confidence = Some(scan_args.alert_min_confidence.into());
}
if user_set(sub_matches, "alert_include_secret") {
defaults.include_secret = Some(scan_args.alert_include_secret);
}
if user_set(sub_matches, "alert_report_url")
&& let Some(u) = &scan_args.alert_report_url
{
defaults.report_url = Some(u.clone());
}
if user_set(sub_matches, "alert_detail") {
defaults.detail = Some(scan_args.alert_detail);
}
if user_set(sub_matches, "alert_finding_filter") {
defaults.finding_filter = Some(scan_args.alert_finding_filter);
}
if user_set(sub_matches, "alert_prevent_empty") {
defaults.prevent_empty = Some(scan_args.alert_prevent_empty);
}
alerts.defaults = defaults;
for url in &scan_args.alert_webhook {
alerts.webhooks.push(WebhookConfig {
url: url.clone(),
format: None,
on: None,
min_confidence: None,
include_secret: None,
report_url: None,
detail: None,
finding_filter: None,
prevent_empty: None,
});
}
cfg.alerts = alerts;
let mut g = GlobalConfig::default();
if user_set(sub_matches, "tls_mode") {
g.tls_mode = Some(global_args.tls_mode.into());
}
if user_set(sub_matches, "allow_internal_ips") {
g.allow_internal_ips = Some(global_args.allow_internal_ips);
}
if user_set(sub_matches, "no_update_check") {
g.no_update_check = Some(global_args.no_update_check);
}
if user_set(sub_matches, "user_agent_suffix")
&& let Some(s) = &global_args.user_agent_suffix
{
g.user_agent_suffix = Some(s.clone());
}
if !global_args.endpoint.is_empty() {
g.endpoints = global_args.endpoint.clone();
}
if user_set(sub_matches, "endpoint_config")
&& let Some(p) = &global_args.endpoint_config
{
g.endpoint_config = Some(p.clone());
}
cfg.global = g;
let mut git = GitConfig::default();
if user_set(sub_matches, "git_clone_dir")
&& let Some(p) = &scan_args.input_specifier_args.git_clone_dir
{
git.clone_dir = Some(p.clone());
}
if user_set(sub_matches, "keep_clones") {
git.keep_clones = Some(scan_args.input_specifier_args.keep_clones);
}
if user_set(sub_matches, "repo_clone_limit")
&& let Some(n) = scan_args.input_specifier_args.repo_clone_limit
{
git.repo_clone_limit = Some(n);
}
if user_set(sub_matches, "include_contributors") {
git.include_contributors = Some(scan_args.input_specifier_args.include_contributors);
}
fn raw_arg_string(matches: &clap::ArgMatches, id: &str) -> Option<String> {
matches.get_raw(id).and_then(|mut v| v.next()).and_then(|s| s.to_str()).map(str::to_owned)
}
if user_set(sub_matches, "github_api_url") {
git.github_api_url = raw_arg_string(sub_matches, "github_api_url");
}
if user_set(sub_matches, "gitlab_api_url") {
git.gitlab_api_url = raw_arg_string(sub_matches, "gitlab_api_url");
}
cfg.git = git;
let mut value =
serde_yaml::to_value(&cfg).context("serialize KingfisherConfig to YAML value")?;
prune_empty(&mut value);
let mut yaml = serde_yaml::to_string(&value).context("emit YAML")?;
if yaml.trim() == "{}" || yaml.trim().is_empty() {
yaml = String::from("# kingfisher.yaml — no flags supplied; nothing to emit.\n");
} else {
let header = "# kingfisher.yaml — generated by `kingfisher config init`.\n\
# Edit freely; CLI flags always override config values.\n";
yaml = format!("{header}{yaml}");
}
Ok(yaml)
}
fn prune_empty(value: &mut serde_yaml::Value) {
use serde_yaml::Value;
match value {
Value::Mapping(map) => {
let keys: Vec<_> = map.keys().cloned().collect();
for k in keys {
if let Some(v) = map.get_mut(&k) {
prune_empty(v);
let drop = match v {
Value::Null => true,
Value::Sequence(s) => s.is_empty(),
Value::Mapping(m) => m.is_empty(),
_ => false,
};
if drop {
map.remove(&k);
}
}
}
}
Value::Sequence(s) => {
for v in s.iter_mut() {
prune_empty(v);
}
}
_ => {}
}
}
fn describe_scan_target(args: &InputSpecifierArgs) -> Option<String> {
fn join_brief<T: std::fmt::Display>(items: &[T], label: &str) -> String {
match items.len() {
0 => String::new(),
1 => items[0].to_string(),
n if n <= 3 => items.iter().map(|i| i.to_string()).collect::<Vec<_>>().join(", "),
n => format!("{} {label}", n),
}
}
if !args.path_inputs.is_empty() {
let s = if args.path_inputs.len() == 1 {
args.path_inputs[0].display().to_string()
} else if args.path_inputs.len() <= 3 {
args.path_inputs.iter().map(|p| p.display().to_string()).collect::<Vec<_>>().join(", ")
} else {
format!("{} paths", args.path_inputs.len())
};
return Some(s);
}
if !args.git_url.is_empty() {
return Some(join_brief(&args.git_url, "git URLs"));
}
if !args.github_event_user.is_empty() {
return Some(format!(
"github public events: {}",
join_brief(&args.github_event_user, "github users")
));
}
if !args.github_user.is_empty() {
return Some(format!("github user: {}", join_brief(&args.github_user, "github users")));
}
if !args.github_organization.is_empty() {
return Some(format!(
"github org: {}",
join_brief(&args.github_organization, "github orgs")
));
}
if args.all_github_organizations {
return Some("all GitHub organizations".to_string());
}
if !args.gitlab_user.is_empty() {
return Some(format!("gitlab user: {}", join_brief(&args.gitlab_user, "gitlab users")));
}
if !args.gitlab_group.is_empty() {
return Some(format!("gitlab group: {}", join_brief(&args.gitlab_group, "gitlab groups")));
}
if !args.huggingface_user.is_empty()
|| !args.huggingface_organization.is_empty()
|| !args.huggingface_model.is_empty()
|| !args.huggingface_dataset.is_empty()
|| !args.huggingface_space.is_empty()
|| !args.huggingface_bucket.is_empty()
{
return Some("huggingface".to_string());
}
if !args.gitea_user.is_empty() || !args.gitea_organization.is_empty() {
return Some("gitea".to_string());
}
if !args.bitbucket_user.is_empty() || !args.bitbucket_workspace.is_empty() {
return Some("bitbucket".to_string());
}
if !args.azure_organization.is_empty() {
return Some(format!("azure: {}", join_brief(&args.azure_organization, "azure orgs")));
}
if let Some(b) = &args.s3_bucket {
return Some(format!("s3://{}{}", b, args.s3_prefix.as_deref().unwrap_or("")));
}
if let Some(b) = &args.gcs_bucket {
return Some(format!("gs://{}{}", b, args.gcs_prefix.as_deref().unwrap_or("")));
}
if !args.docker_image.is_empty() || !args.docker_archive.is_empty() {
let mut docker_targets = Vec::new();
if !args.docker_image.is_empty() {
docker_targets.push(join_brief(&args.docker_image, "images"));
}
if !args.docker_archive.is_empty() {
let archives =
args.docker_archive.iter().map(|p| p.display().to_string()).collect::<Vec<_>>();
docker_targets.push(join_brief(&archives, "archives"));
}
return Some(format!("docker: {}", docker_targets.join(", ")));
}
if let Some(u) = &args.jira_url {
return Some(format!("jira: {}", u));
}
if let Some(u) = &args.confluence_url {
return Some(format!("confluence: {}", u));
}
if args.slack_query.is_some() {
return Some("slack search".to_string());
}
if args.teams_query.is_some() {
return Some("teams search".to_string());
}
if !args.postman_workspaces.is_empty()
|| !args.postman_collections.is_empty()
|| args.postman_all
{
return Some("postman".to_string());
}
None
}
fn should_stage_stdin(input_args: &InputSpecifierArgs, stdin_is_terminal: bool) -> bool {
!stdin_is_terminal && input_args.path_inputs.iter().any(is_stdin_placeholder)
}
fn is_stdin_placeholder(path: impl AsRef<std::path::Path>) -> bool {
path.as_ref().as_os_str() == "-"
}
fn replace_stdin_placeholders(path_inputs: &mut Vec<PathBuf>, stdin_file: PathBuf) {
let mut staged = false;
let mut resolved = Vec::with_capacity(path_inputs.len());
for path in path_inputs.drain(..) {
if is_stdin_placeholder(&path) {
if !std::mem::replace(&mut staged, true) {
resolved.push(stdin_file.clone());
}
} else {
resolved.push(path);
}
}
*path_inputs = resolved;
}
fn build_alert_sinks(
scan_args: &cli::commands::scan::ScanArgs,
) -> Vec<kingfisher::alerts::AlertSink> {
let cli_count =
scan_args.alert_webhook.len().saturating_sub(scan_args.config_webhook_overrides.len());
scan_args
.alert_webhook
.iter()
.enumerate()
.map(|(i, url)| {
let override_ = if i >= cli_count {
scan_args.config_webhook_overrides.get(i - cli_count).cloned().unwrap_or_default()
} else {
cli::commands::scan::ConfigWebhookOverride::default()
};
let format = override_
.format
.or(scan_args.alert_format)
.unwrap_or_else(|| kingfisher::alerts::AlertFormat::infer_from_url(url));
let finding_filter = override_.finding_filter.unwrap_or(scan_args.alert_finding_filter);
kingfisher::alerts::AlertSink {
url: url.clone(),
format,
on: override_.on.unwrap_or(scan_args.alert_on),
min_confidence: override_.min_confidence.unwrap_or(scan_args.alert_min_confidence),
include_secret: override_.include_secret.unwrap_or(scan_args.alert_include_secret),
report_url: override_
.report_url
.clone()
.or_else(|| scan_args.alert_report_url.clone()),
detail: override_.detail.unwrap_or(scan_args.alert_detail),
finding_filter,
prevent_empty: override_.prevent_empty.unwrap_or(scan_args.alert_prevent_empty),
}
})
.collect()
}
fn warn_on_alert_misconfiguration(scan_args: &cli::commands::scan::ScanArgs) {
if scan_args.alert_webhook.is_empty() {
return;
}
for sink in build_alert_sinks(scan_args) {
if sink.finding_filter == kingfisher::alerts::AlertFindingFilter::AccessMapOnly
&& !scan_args.access_map
{
warn!(
"alert sink {} uses access-map-only filtering but --blast-radius was not enabled; \
this sink will not include findings (use --alert-prevent-empty to suppress \
empty filtered alerts)",
kingfisher::alerts::redact_webhook(&sink.url)
);
}
}
}
pub fn determine_exit_code(datastore: &Arc<Mutex<findings_store::FindingsStore>>) -> i32 {
let ds = datastore.lock().unwrap();
let all_matches = ds
.get_matches()
.iter()
.filter(|msg| {
let (_, _, match_item) = &***msg;
match_item.visible
})
.collect::<Vec<_>>();
if all_matches.is_empty() {
0
} else {
let validated_matches = all_matches
.iter()
.filter(|msg| {
let (_, _, match_item) = &****msg;
match_item.rule.syntax().is_authoritative()
&& match_item.validation_outcome.is_verified_active()
})
.count();
if validated_matches > 0 {
205
} else {
200
}
}
}
async fn async_main(args: CommandLineArgs, matches: clap::ArgMatches) -> Result<AsyncMainOutcome> {
setup_logging(&args.global_args);
let global_args = args.global_args.clone();
match args.command {
Command::SelfUpdate => {
let mut g = global_args;
g.self_update = true;
g.no_update_check = false;
let _ = check_for_update_async(&g, None).await;
Ok(AsyncMainOutcome::Done)
}
Command::View(view_args) => view::run(view_args).await.map(|_| AsyncMainOutcome::Done),
Command::BlastRadius(blast_radius_args) => {
if blast_radius_args.rule.is_none() {
let view_report = blast_radius_args.view_report;
let provider = blast_radius_args
.input
.as_deref()
.ok_or_else(|| {
anyhow::anyhow!("a provider is required for standalone mapping")
})?
.parse::<cli::commands::access_map::AccessMapProvider>()
.map_err(|error| anyhow::anyhow!(error))?;
let format = match blast_radius_args.format.as_str() {
"json" => cli::commands::access_map::AccessMapOutputFormat::Json,
"html" => cli::commands::access_map::AccessMapOutputFormat::Html,
_ if blast_radius_args.view_report => {
cli::commands::access_map::AccessMapOutputFormat::Json
}
_ => anyhow::bail!(
"standalone blast-radius mapping supports only json and html output"
),
};
let access_map_args = cli::commands::access_map::AccessMapArgs {
provider,
credential_path: blast_radius_args.credential_path,
output_args: cli::commands::access_map::AccessMapOutputArgs {
output: blast_radius_args.output,
format,
},
};
if view_report {
let result = access_map::map_credential(&access_map_args).await?;
let report_bytes = direct_access_map::build_viewer_report_bytes(&[
direct_access_map::DirectAccessMapResult {
rule_id: format!("standalone.{}", result.cloud),
rule_name: format!("Standalone {} credential", result.cloud),
result,
},
])?;
view::run(view::ViewArgs {
reports: Vec::new(),
port: view::DEFAULT_PORT,
address: view::DEFAULT_ADDRESS.to_string(),
open_browser: true,
report_bytes: Some(report_bytes),
})
.await?;
} else {
access_map::run(access_map_args).await?;
}
return Ok(AsyncMainOutcome::Done);
}
let results =
direct_access_map::run_direct_access_map(&blast_radius_args, &global_args).await?;
if blast_radius_args.view_report {
let report_bytes = direct_access_map::build_viewer_report_bytes(&results)?;
view::run(view::ViewArgs {
reports: Vec::new(),
port: view::DEFAULT_PORT,
address: view::DEFAULT_ADDRESS.to_string(),
open_browser: true,
report_bytes: Some(report_bytes),
})
.await?;
} else {
direct_access_map::print_results(
&results,
&blast_radius_args.format,
blast_radius_args.output.as_deref(),
)?;
}
Ok(AsyncMainOutcome::Done)
}
Command::Config(config_args) => {
run_config_command(config_args, &global_args, &matches)?;
Ok(AsyncMainOutcome::Done)
}
Command::Validate(validate_args) => {
let results =
direct_validate::run_direct_validation(&validate_args, &global_args).await?;
let use_color = global_args.use_color(std::io::stdout());
direct_validate::print_results(&results, &validate_args.format, use_color);
if direct_validate::any_actionable(&results) {
Ok(AsyncMainOutcome::Done)
} else {
std::process::exit(1);
}
}
Command::Revoke(revoke_args) => {
let results = direct_revoke::run_direct_revocation(&revoke_args, &global_args).await?;
let use_color = global_args.use_color(std::io::stdout());
direct_revoke::print_results(&results, &revoke_args.format, use_color);
if direct_revoke::any_revoked(&results) {
Ok(AsyncMainOutcome::Done)
} else {
std::process::exit(1);
}
}
command => {
let update_status = check_for_update_async(&global_args, None).await;
if update_status.was_self_updated {
return Ok(AsyncMainOutcome::Reexec);
}
match command {
Command::Scan(scan_command) => match scan_command.into_operation()? {
ScanOperation::Scan(mut scan_args) => {
let loaded_config = load_project_config(global_args.config.as_deref())?;
let scan_matches = matches.subcommand_matches("scan");
let mut effective_global_args = global_args.clone();
if let Some(cfg) = &loaded_config {
apply_config(
&mut scan_args,
&mut effective_global_args,
cfg,
scan_matches,
);
set_user_agent_suffix(effective_global_args.user_agent_suffix.clone());
}
let global_args = effective_global_args;
scan_args.validate_audit_log_collisions(
global_args.endpoint_config.as_deref(),
global_args.config.as_deref(),
)?;
warn_on_alert_misconfiguration(&scan_args);
if scan_args.view_report {
view::ensure_port_available(
scan_args.view_report_port,
&scan_args.view_report_address,
"--view-report-port",
)?;
}
let view_scan_started_at = chrono::Local::now();
let view_scan_start_time = Instant::now();
let temp_dir =
TempDir::new().context("Failed to create temporary directory")?;
let temp_dir_path = temp_dir.path().to_path_buf();
let clone_dir = if let Some(clone_dir) =
scan_args.input_specifier_args.git_clone_dir.as_ref()
{
std::fs::create_dir_all(clone_dir)?;
clone_dir.to_path_buf()
} else {
temp_dir_path.clone()
};
let keep_clones = scan_args.input_specifier_args.keep_clones
&& scan_args.input_specifier_args.git_clone_dir.is_none();
let auto_cleanup_clones = !scan_args.input_specifier_args.keep_clones
&& scan_args.input_specifier_args.git_clone_dir.is_none();
let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir)));
info!(
"Launching with {} concurrent scan jobs. Use --jobs to override.",
&scan_args.num_jobs
);
if should_stage_stdin(
&scan_args.input_specifier_args,
std::io::stdin().is_terminal(),
) {
let mut buf = Vec::new();
std::io::stdin().read_to_end(&mut buf)?;
let stdin_file = temp_dir_path.join("stdin_input");
std::fs::write(&stdin_file, buf)?;
replace_stdin_placeholders(
&mut scan_args.input_specifier_args.path_inputs,
stdin_file,
);
}
let rules_db = Arc::new(load_and_record_rules(
&scan_args,
&datastore,
global_args.use_progress(),
)?);
run_scan(
&global_args,
&scan_args,
&rules_db,
Arc::clone(&datastore),
&update_status,
auto_cleanup_clones,
)
.await?;
if update_status.is_outdated
&& let Some(styled) = &update_status.styled_message
{
let _ = writeln!(std::io::stderr(), "{}", styled);
}
let exit_code = determine_exit_code(&datastore);
if !scan_args.alert_webhook.is_empty() {
let alert_reporter = DetailsReporter {
datastore: Arc::clone(&datastore),
styles: Styles::new(global_args.use_color(std::io::stdout())),
validation_filter: scan_args.effective_validation_filter(),
audit_context: None,
};
match alert_reporter.build_finding_records(&scan_args) {
Ok(records) => {
let target =
describe_scan_target(&scan_args.input_specifier_args);
let access_map =
alert_reporter.build_alert_access_map_entries(&scan_args);
let sinks: Vec<_> = build_alert_sinks(&scan_args)
.into_iter()
.filter(|sink| {
match kingfisher::alerts::validate_webhook_url(
&sink.url,
) {
Ok(()) => true,
Err(e) => {
warn!("alert dispatch: skipping sink: {}", e);
false
}
}
})
.collect();
kingfisher::alerts::dispatch_with_context(
&sinks,
&records,
&access_map,
target,
scan_args.alert_dry_run,
)
.await;
}
Err(e) => warn!("alert dispatch: failed to build findings: {}", e),
}
}
if scan_args.view_report {
let audit_context = ScanAuditContext {
scan_timestamp: Some(view_scan_started_at.to_rfc3339()),
scan_duration_seconds: Some(
view_scan_start_time.elapsed().as_secs_f64(),
),
rules_applied: Some(rules_db.num_rules()),
successful_validations: None,
failed_validations: None,
skipped_validations: None,
blobs_scanned: None,
bytes_scanned: None,
running_version: Some(update_status.running_version.clone()),
latest_version: update_status.latest_version.clone(),
update_check_status: Some(
update_status.check_status.as_str().to_string(),
),
};
let reporter = DetailsReporter {
datastore: Arc::clone(&datastore),
styles: Styles::new(global_args.use_color(std::io::stdout())),
validation_filter: scan_args.effective_validation_filter(),
audit_context: Some(audit_context),
};
let envelope = reporter.build_report_envelope(&scan_args)?;
let report_bytes = serde_json::to_vec_pretty(&envelope)?;
let view_args = view::ViewArgs {
reports: vec![],
port: scan_args.view_report_port,
address: scan_args.view_report_address.clone(),
open_browser: true,
report_bytes: Some(report_bytes),
};
view::run(view_args).await?;
}
if keep_clones {
let _kept_path = temp_dir.keep(); } else if let Err(e) = temp_dir.close() {
eprintln!("Failed to close temporary directory: {}", e);
}
std::process::exit(exit_code);
}
ScanOperation::ListRepositories(list_command) => match list_command {
ListRepositoriesCommand::Github { api_url, specifiers } => {
github::list_repositories(
api_url,
global_args.ignore_certs,
global_args.use_progress(),
&specifiers.user,
specifiers.include_gists,
&specifiers.organization,
specifiers.all_organizations,
&specifiers.exclude_repos,
specifiers.repo_type.into(),
)
.await?;
}
ListRepositoriesCommand::Gitlab { api_url, specifiers } => {
kingfisher::gitlab::list_repositories(
api_url,
global_args.ignore_certs,
global_args.use_progress(),
&specifiers.user,
specifiers.include_snippets,
&specifiers.group,
specifiers.all_groups,
specifiers.include_subgroups,
&specifiers.exclude_repos,
specifiers.repo_type.into(),
)
.await?;
}
ListRepositoriesCommand::Gitea { api_url, specifiers } => {
gitea::list_repositories(
api_url,
global_args.ignore_certs,
global_args.use_progress(),
&specifiers.user,
&specifiers.organization,
specifiers.all_organizations,
&specifiers.exclude_repos,
specifiers.repo_type.into(),
)
.await?;
}
ListRepositoriesCommand::Bitbucket { api_url, specifiers } => {
let auth_config = bitbucket::AuthConfig::from_env();
bitbucket::list_repositories(
api_url,
auth_config,
global_args.ignore_certs,
global_args.use_progress(),
&specifiers.user,
specifiers.include_snippets,
&specifiers.workspace,
&specifiers.project,
specifiers.all_workspaces,
&specifiers.exclude_repos,
specifiers.repo_type.into(),
)
.await?;
}
ListRepositoriesCommand::Azure { base_url, specifiers } => {
azure::list_repositories(
base_url,
global_args.ignore_certs,
global_args.use_progress(),
&specifiers.organization,
&specifiers.project,
specifiers.all_projects,
&specifiers.exclude_repos,
specifiers.repo_type.into(),
)
.await?;
}
ListRepositoriesCommand::Huggingface { specifiers } => {
let repo_specifiers = huggingface::RepoSpecifiers {
user: specifiers.user.clone(),
organization: specifiers.organization.clone(),
model: specifiers.model.clone(),
dataset: specifiers.dataset.clone(),
space: specifiers.space.clone(),
bucket: specifiers.bucket.clone(),
exclude: specifiers.exclude.clone(),
};
let auth = huggingface::AuthConfig::from_env();
huggingface::list_repositories(
&repo_specifiers,
&auth,
global_args.ignore_certs,
global_args.use_progress(),
)
.await?;
}
},
},
Command::Rules(ref rule_args) => match &rule_args.command {
RulesCommand::Check(check_args) => {
run_rules_check(check_args)?;
}
RulesCommand::CompileCache(cache_args) => {
run_rules_compile_cache(cache_args)?;
}
RulesCommand::PruneCache(prune_args) => {
run_rules_prune_cache(prune_args)?;
}
RulesCommand::List(list_args) => {
run_rules_list(list_args)?;
}
},
Command::View(_) | Command::Wizard(_) => {
anyhow::bail!("View and wizard commands should not reach this branch")
}
Command::BlastRadius(_) => {
anyhow::bail!("BlastRadius command should not reach this branch")
}
Command::Validate(_) => {
anyhow::bail!("Validate command should not reach this branch")
}
Command::Revoke(_) => {
anyhow::bail!("Revoke command should not reach this branch")
}
Command::SelfUpdate => {
anyhow::bail!("SelfUpdate command should not reach this branch")
}
Command::Config(_) => {
anyhow::bail!("Config command should not reach this branch")
}
}
if let Some(message) = &update_status.message {
info!("{}", message);
}
Ok(AsyncMainOutcome::Done)
}
}
}
fn create_default_scan_args() -> cli::commands::scan::ScanArgs {
use cli::commands::scan::*;
ScanArgs {
num_jobs: 1,
rules: RuleSpecifierArgs {
rules_path: Vec::new(),
rule: vec!["all".into()],
exclude_rule: Vec::new(),
load_builtins: true,
},
rule_cache: RuleCacheArgs::default(),
input_specifier_args: InputSpecifierArgs {
path_inputs: Vec::new(),
git_url: Vec::new(),
git_clone_dir: None,
keep_clones: false,
repo_clone_limit: None,
include_contributors: false,
github_user: Vec::new(),
github_include_gists: false,
github_organization: Vec::new(),
github_exclude: Vec::new(),
all_github_organizations: false,
github_api_url: url::Url::parse("https://api.github.com/").unwrap(),
github_repo_type: GitHubRepoType::Source,
github_event_user: Vec::new(),
github_event_lookback_hours: 24,
gitlab_user: Vec::new(),
gitlab_include_snippets: false,
gitlab_group: Vec::new(),
gitlab_exclude: Vec::new(),
all_gitlab_groups: false,
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
gitlab_repo_type: GitLabRepoType::All,
gitlab_include_subgroups: false,
huggingface_user: Vec::new(),
huggingface_organization: Vec::new(),
huggingface_model: Vec::new(),
huggingface_dataset: Vec::new(),
huggingface_space: Vec::new(),
huggingface_bucket: Vec::new(),
huggingface_exclude: Vec::new(),
gitea_user: Vec::new(),
gitea_organization: Vec::new(),
gitea_exclude: Vec::new(),
all_gitea_organizations: false,
gitea_api_url: Url::parse("https://gitea.com/api/v1/").unwrap(),
gitea_repo_type: GiteaRepoType::Source,
bitbucket_user: Vec::new(),
bitbucket_include_snippets: false,
bitbucket_workspace: Vec::new(),
bitbucket_project: Vec::new(),
bitbucket_exclude: Vec::new(),
all_bitbucket_workspaces: false,
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
bitbucket_repo_type: BitbucketRepoType::Source,
bitbucket_auth: BitbucketAuthArgs::default(),
azure_organization: Vec::new(),
azure_project: Vec::new(),
azure_exclude: Vec::new(),
all_azure_projects: false,
azure_base_url: Url::parse("https://dev.azure.com/").unwrap(),
azure_repo_type: AzureRepoType::Source,
jira_url: None,
jql: None,
jira_include_comments: false,
jira_include_changelog: false,
confluence_url: None,
cql: None,
max_results: 100,
s3_bucket: None,
s3_prefix: None,
role_arn: None,
aws_local_profile: None,
gcs_bucket: None,
gcs_prefix: None,
gcs_service_account: None,
slack_query: None,
slack_api_url: Url::parse("https://slack.com/api/").unwrap(),
teams_query: None,
teams_api_url: Url::parse("https://graph.microsoft.com/").unwrap(),
postman_workspaces: Vec::new(),
postman_collections: Vec::new(),
postman_environments: Vec::new(),
postman_all: false,
postman_include_mocks_monitors: false,
postman_api_url: Url::parse("https://api.getpostman.com/").unwrap(),
docker_image: Vec::new(),
docker_archive: Vec::new(),
git_clone: GitCloneMode::Bare,
git_history: GitHistoryMode::Full,
commit_metadata: true,
repo_artifacts: false,
scan_nested_repos: true,
since_commit: None,
branch: None,
branch_root: false,
branch_root_commit: None,
staged: false,
},
extra_ignore_comments: Vec::new(),
content_filtering_args: ContentFilteringArgs {
max_file_size_mb: 25.0,
no_extract_archives: true,
extraction_depth: 2,
exclude: Vec::new(), no_binary: true,
},
confidence: ConfidenceLevel::Medium,
disk_offload: false,
no_validate: true,
access_map: false,
rule_stats: false,
only_valid: false,
validation_filter: None,
include_hidden_findings: false,
min_entropy: None,
redact: false,
git_repo_timeout: 1800,
audit_log: None,
no_dedup: false,
view_report: false,
baseline_file: None,
manage_baseline: false,
skip_regex: Vec::new(),
skip_word: Vec::new(),
skip_aws_account: Vec::new(),
skip_aws_account_file: None,
output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty },
no_base64: false,
turbo: false,
no_inline_ignore: false,
no_ignore_if_contains: false,
view_report_port: view::DEFAULT_PORT,
view_report_address: view::DEFAULT_ADDRESS.to_string(),
validation_timeout: 10,
validation_retries: 1,
validation_rps: None,
validation_rps_rule: Vec::new(),
full_validation_response: false,
max_validation_response_length: 2048,
alert_webhook: Vec::new(),
alert_format: None,
alert_on: kingfisher::alerts::AlertOn::Findings,
alert_min_confidence: kingfisher::cli::commands::scan::ConfidenceLevel::Medium,
alert_include_secret: false,
alert_report_url: None,
alert_detail: kingfisher::alerts::AlertDetail::Auto,
alert_finding_filter: kingfisher::alerts::AlertFindingFilter::All,
alert_prevent_empty: false,
alert_dry_run: false,
config_webhook_overrides: Vec::new(),
}
}
pub fn run_rules_compile_cache(args: &RulesCompileCacheArgs) -> Result<()> {
let mut scan_args = create_default_scan_args();
scan_args.confidence = args.confidence;
let loader = RuleLoader::from_rule_specifiers(&args.rules);
let loaded = loader.load(&scan_args).context("Failed to load rules")?;
let resolved = loaded.resolve_enabled_rules_owned().context("Failed to resolve rules")?;
let betterleaks_prefilter = loaded.betterleaks_prefilter_for(&resolved);
let cache = RuleCacheConfig::from_dir_or_env(args.cache.rule_cache_dir.clone());
info!(cache_dir = %cache.cache_dir().display(), "Using Vectorscan rule cache");
let rules_db = RulesDatabase::from_rules_with_cache_and_betterleaks_prefilter(
resolved,
&cache,
betterleaks_prefilter,
)
.context("Failed to compile rules with Vectorscan cache")?;
println!("Rule cache ready: {} rules in {}", rules_db.num_rules(), cache.cache_dir().display());
Ok(())
}
pub fn run_rules_prune_cache(args: &RuleCachePruneArgs) -> Result<()> {
let cache = RuleCacheConfig::from_dir_or_env(args.cache.rule_cache_dir.clone());
let summary = prune_rule_cache(
&cache,
&RuleCachePruneConfig {
max_entries: args.max_entries,
max_age: args.max_age,
protected_cache_key: None,
dry_run: args.dry_run,
},
);
let action = if args.dry_run { "would remove" } else { "removed" };
println!(
"Rule cache prune {action} {} entries ({} bytes) from {}; scanned {} entries, {} valid, {} invalid, {} protected, {} removal errors",
if args.dry_run { summary.candidate_entries } else { summary.removed_entries },
if args.dry_run { summary.candidate_bytes } else { summary.removed_bytes },
cache.cache_dir().display(),
summary.scanned_entries,
summary.valid_entries,
summary.invalid_entries,
summary.protected_entries,
summary.removal_errors
);
Ok(())
}
pub fn run_rules_check(args: &RulesCheckArgs) -> Result<()> {
let mut num_errors = 0;
let mut num_warnings = 0;
let loader = RuleLoader::from_rule_specifiers(&args.rules);
let loaded = loader.load(&create_default_scan_args())?;
let resolved = loaded.resolve_enabled_rules_owned()?;
let betterleaks_prefilter = loaded.betterleaks_prefilter_for(&resolved);
let rules_db =
RulesDatabase::from_rules_with_betterleaks_prefilter(resolved, betterleaks_prefilter)?;
for (rule_index, rule) in rules_db.rules().iter().enumerate() {
let rule_syntax = rule.syntax();
if rule.name().len() < 3 {
warn!("Rule '{}' has a very short name", rule.name());
num_warnings += 1;
}
if rule.syntax().pattern.len() < 5 {
warn!("Rule '{}' has a very short pattern", rule.name());
num_warnings += 1;
}
if rule.syntax().examples.is_empty() {
if !rule.id().starts_with("betterleaks.") {
warn!("Rule '{}' has no examples", rule.name());
num_warnings += 1;
}
continue;
}
if let Err(e) = rule.syntax().as_regex() {
error!("Rule '{}' has invalid regex: {}", rule.name(), e);
num_errors += 1;
continue;
}
for (example_index, example) in rule_syntax.examples.iter().enumerate() {
let re =
rules_db.get_regex_by_rule_id(rule.id()).expect("Failed to get regex for rule");
let example_bytes = example.as_bytes();
let regex_matched = re.is_match(example_bytes);
if !regex_matched {
println!("\nTesting rule {} - {}", rule_index + 1, rule_syntax.name);
println!(" Processing example {}", example_index + 1);
println!(" [!] Pattern mismatch detected for example: {}", example);
println!(" Regex match: {}", regex_matched);
num_errors += 1;
continue;
}
if let Some(pattern_reqs) = rule.pattern_requirements() {
if let Some(captures) = re.captures(example_bytes) {
let full_capture = captures.get(0).expect("Group 0 should always exist");
let full_bytes = full_capture.as_bytes();
let matching_input_for_validation = 'block: {
if let Some(secret_cap) =
captures.name("secret").or_else(|| captures.name("SECRET"))
{
break 'block secret_cap;
}
if let Some(named_cap) = (1..captures.len()).find_map(|i| {
let name_opt = re.capture_names().nth(i).and_then(|n| n);
name_opt.and_then(|_| captures.get(i))
}) {
break 'block named_cap;
}
if let Some(pos_cap) = captures.get(1) {
break 'block pos_cap;
}
break 'block full_capture;
};
let validation_bytes = matching_input_for_validation.as_bytes();
use kingfisher_rules::PatternRequirementContext;
let context = PatternRequirementContext {
regex: re,
captures: &captures,
full_match: full_bytes,
};
use kingfisher_rules::PatternValidationResult;
match pattern_reqs.validate(validation_bytes, Some(context), false) {
PatternValidationResult::Passed => {
}
PatternValidationResult::Failed => {
println!("\nTesting rule {} - {}", rule_index + 1, rule_syntax.name);
println!(" Processing example {}", example_index + 1);
println!(
" [!] Pattern requirements not met for example: {}",
example
);
println!(
" The match does not satisfy the character requirements (min_digits, min_uppercase, etc.)"
);
num_errors += 1;
}
PatternValidationResult::FailedChecksum { actual_len, expected_len } => {
println!("\nTesting rule {} - {}", rule_index + 1, rule_syntax.name);
println!(" Processing example {}", example_index + 1);
println!(" [!] Checksum validation failed for example: {}", example);
println!(
" Actual checksum length: {}, Expected checksum length: {}",
actual_len, expected_len
);
num_errors += 1;
}
PatternValidationResult::IgnoredBySubstring { matched_term } => {
println!("\nTesting rule {} - {}", rule_index + 1, rule_syntax.name);
println!(" Processing example {}", example_index + 1);
println!(
" [!] Example would be ignored due to containing term: {}",
matched_term
);
println!(" Example: {}", example);
num_warnings += 1;
}
}
}
}
}
}
if num_errors > 0 || num_warnings > 0 {
println!("\nCheck Summary:");
println!(" Errors: {}", num_errors);
println!(" Warnings: {}", num_warnings);
println!("\nError types include:");
println!(" - Invalid regex patterns");
println!(" - Examples that don't match their patterns");
println!("\nWarning types include:");
println!(" - Rules with very short names");
println!(" - Rules with very short patterns");
println!(" - Rules without examples");
} else {
println!("\nAll rules passed validation successfully!");
}
if num_errors > 0 || (args.warnings_as_errors && num_warnings > 0) {
std::process::exit(1);
}
Ok(())
}
pub fn run_rules_list(args: &RulesListArgs) -> Result<()> {
let loader = RuleLoader::from_rule_specifiers(&args.rules);
let loaded = loader.load(&create_default_scan_args())?;
let resolved = loaded.resolve_enabled_rules()?;
let mut writer = args.output_args.get_writer()?;
#[cfg(debug_assertions)]
let show_validation = args.show_validation;
#[cfg(not(debug_assertions))]
let show_validation = false;
match args.output_args.format {
RulesListOutputFormat::Pretty => {
let term_width = usize::from(Term::stdout().size().1);
let max_name_width = resolved.iter().map(|r| r.name().len()).max().unwrap_or(0).max(4); let max_id_width = resolved.iter().map(|r| r.id().len()).max().unwrap_or(0).max(2); let max_conf_width = resolved
.iter()
.map(|r| format!("{:?}", r.confidence()).len())
.max()
.unwrap_or(0)
.max(10); let reserved_width = max_name_width + max_id_width + max_conf_width + 10;
let pattern_width = term_width.saturating_sub(reserved_width);
let format_pattern = |pattern: &str| {
let single_line = pattern
.replace(['\n', '\r'], " ")
.split_whitespace()
.collect::<Vec<_>>()
.join(" ");
if single_line.len() > pattern_width {
format!("{}...", &single_line[..pattern_width.saturating_sub(3)])
} else {
single_line
}
};
writeln!(
writer,
"\n{:name_width$} │ {:id_width$} │ {:conf_width$} │ Pattern",
"Rule",
"ID",
"Confidence",
name_width = max_name_width,
id_width = max_id_width,
conf_width = max_conf_width
)?;
writeln!(
writer,
"{0:─<name_width$} ┼ {0:─<id_width$} ┼ {0:─<conf_width$} ┼ {0:─<pattern_width$}",
"",
name_width = max_name_width,
id_width = max_id_width,
conf_width = max_conf_width,
pattern_width = pattern_width
)?;
for rule in resolved {
let formatted_pattern = format_pattern(&rule.syntax().pattern);
writeln!(
writer,
"{:name_width$} │ {:id_width$} │ {:conf_width$} │ {}",
rule.name(),
rule.id(),
format!("{:?}", rule.confidence()),
formatted_pattern,
name_width = max_name_width,
id_width = max_id_width,
conf_width = max_conf_width
)?;
if show_validation && let Some(validation) = &rule.syntax().validation {
match validation {
kingfisher::rules::Validation::Betterleaks(validation) => {
#[cfg(debug_assertions)]
writeln!(writer, " Validation: {}", validation.source)?;
writeln!(writer, " Validation AST: {:?}", validation.expression)?;
}
validation => writeln!(writer, " Validation: {validation:?}")?,
}
}
}
writeln!(writer)?;
}
RulesListOutputFormat::Json => {
let rules_json: Vec<_> = resolved
.iter()
.map(|rule| {
let mut value = json!({
"name": rule.name(),
"id": rule.id(),
"pattern": rule.syntax().pattern,
"confidence": rule.confidence(),
"examples": rule.syntax().examples,
"visible": rule.visible(),
});
if show_validation {
value["validation"] = serde_json::to_value(&rule.syntax().validation)
.expect("validation serialization should succeed");
}
value
})
.collect();
serde_json::to_writer_pretty(&mut writer, &rules_json)?;
writeln!(writer)?;
}
}
Ok(())
}
#[cfg(test)]
mod apply_config_tests {
use std::path::PathBuf;
use clap::{ArgMatches, CommandFactory, FromArgMatches};
use kingfisher::cli::CommandLineArgs;
use kingfisher::cli::commands::output::ReportOutputFormat;
use kingfisher::cli::commands::scan::{ConfidenceLevel, ScanOperation};
use kingfisher::cli::config::{KingfisherConfig, parse_str};
use kingfisher::cli::global::Command;
fn parse(argv: &[&str]) -> (CommandLineArgs, ArgMatches) {
let matches =
CommandLineArgs::command().try_get_matches_from(argv).expect("argv should parse");
let args = CommandLineArgs::from_arg_matches(&matches).unwrap();
(args, matches)
}
fn into_scan(args: CommandLineArgs) -> kingfisher::cli::commands::scan::ScanArgs {
let cmd = match args.command {
Command::Scan(c) => c,
_ => panic!("expected scan subcommand"),
};
match cmd.into_operation().unwrap() {
ScanOperation::Scan(s) => s,
ScanOperation::ListRepositories(_) => panic!("expected scan op"),
}
}
#[test]
fn non_interactive_git_url_does_not_stage_stdin() {
let (args, _) = parse(&["kingfisher", "scan", "github.com/octocat/Hello-World"]);
let scan_args = into_scan(args);
assert!(scan_args.input_specifier_args.path_inputs.is_empty());
assert!(!scan_args.input_specifier_args.git_url.is_empty());
assert!(!super::should_stage_stdin(&scan_args.input_specifier_args, false));
}
#[test]
fn non_interactive_path_target_does_not_stage_stdin() {
let (args, _) = parse(&["kingfisher", "scan", "."]);
let scan_args = into_scan(args);
assert!(!super::should_stage_stdin(&scan_args.input_specifier_args, false));
}
#[test]
fn dash_stages_stdin_only_when_stdin_is_redirected() {
let (args, _) = parse(&["kingfisher", "scan", "-"]);
let scan_args = into_scan(args);
assert!(super::should_stage_stdin(&scan_args.input_specifier_args, false));
assert!(!super::should_stage_stdin(&scan_args.input_specifier_args, true));
}
#[test]
fn github_event_user_file_cannot_alias_audit_log() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("users.txt");
let path = path.to_str().unwrap();
let (args, _) = parse(&[
"kingfisher",
"scan",
"github",
"--public-events",
"--user-file",
path,
"--audit-log",
path,
]);
let command = match args.command {
Command::Scan(command) => command,
_ => panic!("expected scan subcommand"),
};
assert!(command.into_operation().is_err());
}
#[test]
fn staging_stdin_keeps_sibling_paths_and_collapses_repeats() {
let stdin_file = PathBuf::from("/tmp/kf/stdin_input");
let mut path_inputs = vec![
PathBuf::from("-"),
PathBuf::from("./src"),
PathBuf::from("-"),
PathBuf::from("./tests"),
];
super::replace_stdin_placeholders(&mut path_inputs, stdin_file.clone());
assert_eq!(path_inputs, vec![stdin_file, PathBuf::from("./src"), PathBuf::from("./tests")]);
}
#[test]
fn config_wins_when_cli_uses_default() {
let yaml = r#"
scan:
confidence: high
redact: true
output:
format: json
"#;
let cfg: KingfisherConfig = parse_str(yaml).unwrap();
let (args, matches) = parse(&["kingfisher", "scan", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(scan_args.confidence, ConfidenceLevel::High);
assert!(scan_args.redact);
assert_eq!(scan_args.output_args.format, ReportOutputFormat::Json);
}
#[test]
fn config_output_path_colliding_with_audit_log_is_rejected() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("report.json");
let path_str = path.display().to_string();
let yaml = format!(
r#"
output:
path: {path_str}
"#
);
let cfg: KingfisherConfig = parse_str(&yaml).unwrap();
let (args, matches) = parse(&["kingfisher", "scan", "--audit-log", &path_str, "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
assert!(scan_args.validate_audit_log_collisions(None, None).is_ok());
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(scan_args.output_args.output.as_deref(), Some(path.as_path()));
assert!(scan_args.validate_audit_log_collisions(None, None).is_err());
}
#[test]
fn audit_log_colliding_with_baseline_is_rejected() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("baseline-file.yaml");
let path_str = path.display().to_string();
let (args, _) = parse(&[
"kingfisher",
"scan",
"--audit-log",
&path_str,
"--baseline-file",
&path_str,
".",
]);
let cmd = match args.command {
Command::Scan(c) => c,
_ => panic!("expected scan subcommand"),
};
assert!(cmd.into_operation().is_err());
let (args, _) = parse(&[
"kingfisher",
"scan",
"--audit-log",
"baseline-file.yaml",
"--manage-baseline",
".",
]);
let cmd = match args.command {
Command::Scan(c) => c,
_ => panic!("expected scan subcommand"),
};
assert!(cmd.into_operation().is_err());
let (args, _) = parse(&["kingfisher", "scan", "--audit-log", &path_str, "."]);
let scan_args = into_scan(args);
assert!(scan_args.validate_audit_log_collisions(None, None).is_ok());
}
#[test]
fn audit_log_tilde_path_is_expanded_for_collision_checks() {
let expanded =
kingfisher::util::expand_tilde(std::path::Path::new("~/kf-audit-collision.jsonl"));
let input = tempfile::tempdir().unwrap();
let (args, _) = parse(&[
"kingfisher",
"scan",
"--audit-log=~/kf-audit-collision.jsonl",
"--output",
expanded.to_str().unwrap(),
input.path().to_str().unwrap(),
]);
let cmd = match args.command {
Command::Scan(c) => c,
_ => panic!("expected scan subcommand"),
};
assert!(cmd.into_operation().is_err());
}
#[test]
fn audit_log_colliding_with_endpoint_config_is_rejected() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("endpoints.yaml");
let path_str = path.display().to_string();
let (args, _) = parse(&["kingfisher", "scan", "--audit-log", &path_str, "."]);
let scan_args = into_scan(args);
assert!(scan_args.validate_audit_log_collisions(None, None).is_ok());
assert!(scan_args.validate_audit_log_collisions(Some(&path), None).is_err());
}
#[test]
fn audit_log_colliding_with_project_config_is_rejected() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("kingfisher.yaml");
let path_str = path.display().to_string();
let (args, _) = parse(&["kingfisher", "scan", "--audit-log", &path_str, "."]);
let scan_args = into_scan(args);
assert!(scan_args.validate_audit_log_collisions(None, None).is_ok());
assert!(scan_args.validate_audit_log_collisions(None, Some(&path)).is_err());
}
#[test]
fn cli_beats_config_for_scalars() {
let yaml = r#"
scan:
confidence: high
redact: true
output:
format: json
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) =
parse(&["kingfisher", "scan", "--confidence", "low", "--format", "toon", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(scan_args.confidence, ConfidenceLevel::Low);
assert_eq!(scan_args.output_args.format, ReportOutputFormat::Toon);
assert!(scan_args.redact);
}
#[test]
fn lists_are_concatenated_with_cli() {
let yaml = r#"
filters:
skip_words: ["FROM_CONFIG"]
exclude: ["vendor/"]
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) = parse(&[
"kingfisher",
"scan",
"--skip-word",
"FROM_CLI",
"--exclude",
"node_modules/",
".",
]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert!(scan_args.skip_word.contains(&"FROM_CLI".to_string()));
assert!(scan_args.skip_word.contains(&"FROM_CONFIG".to_string()));
assert!(scan_args.content_filtering_args.exclude.contains(&"vendor/".to_string()));
assert!(scan_args.content_filtering_args.exclude.contains(&"node_modules/".to_string()));
}
#[test]
fn rules_enabled_replaces_default_but_appends_to_user_selection() {
let yaml = r#"
rules:
enabled: ["custom"]
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) = parse(&["kingfisher", "scan", "--rule", "default", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert!(scan_args.rules.rule.contains(&"default".to_string()));
assert!(scan_args.rules.rule.contains(&"custom".to_string()));
let (args, matches) = parse(&["kingfisher", "scan", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(scan_args.rules.rule, vec!["custom".to_string()]);
}
#[test]
fn rules_disabled_is_concatenated_with_cli_exclusions() {
let yaml = r#"
rules:
disabled: ["betterleaks.github-pat"]
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) =
parse(&["kingfisher", "scan", "--exclude-rule", "betterleaks.openai-api-key", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(
scan_args.rules.exclude_rule,
vec!["betterleaks.openai-api-key".to_string(), "betterleaks.github-pat".to_string()]
);
}
#[test]
fn cli_rule_and_exclude_rule_flags_are_repeated_and_preserved() {
let (args, matches) = parse(&[
"kingfisher",
"scan",
"--rule",
"betterleaks.github-pat",
"--rule",
"betterleaks.github-fine-grained-pat",
"--exclude-rule",
"betterleaks.openai-api-key",
"--exclude-rule",
"custom.openai.secondary",
".",
]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&kingfisher::cli::config::KingfisherConfig::default(),
matches.subcommand_matches("scan"),
);
assert_eq!(
scan_args.rules.rule,
vec![
"betterleaks.github-pat".to_string(),
"betterleaks.github-fine-grained-pat".to_string(),
]
);
assert_eq!(
scan_args.rules.exclude_rule,
vec!["betterleaks.openai-api-key".to_string(), "custom.openai.secondary".to_string()]
);
}
#[test]
fn rule_cache_config_and_cli_precedence_respects_opt_out() {
let cfg = parse_str(
r#"
rules:
cache: false
"#,
)
.unwrap();
let (args, matches) = parse(&["kingfisher", "scan", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert!(!scan_args.rule_cache.enabled(), "config rules.cache=false should disable cache");
let cfg = parse_str(
r#"
rules:
cache: true
"#,
)
.unwrap();
let (args, matches) = parse(&["kingfisher", "scan", "--no-rule-cache", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert!(!scan_args.rule_cache.enabled(), "CLI --no-rule-cache should beat config");
let cfg = parse_str(
r#"
rules:
cache: false
"#,
)
.unwrap();
let (args, matches) = parse(&["kingfisher", "scan", "--rule-cache", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert!(scan_args.rule_cache.enabled(), "CLI --rule-cache should beat config");
}
#[test]
fn validation_rps_per_rule_appended_as_strings() {
let yaml = r#"
validation:
rps_per_rule:
betterleaks.aws: 1.5
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) =
parse(&["kingfisher", "scan", "--validation-rps-rule", "betterleaks.gcp=2.0", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert!(scan_args.validation_rps_rule.contains(&"betterleaks.gcp=2.0".to_string()));
assert!(scan_args.validation_rps_rule.contains(&"betterleaks.aws=1.5".to_string()));
}
#[test]
fn alerts_defaults_set_alert_globals_when_cli_default() {
let yaml = r#"
alerts:
defaults:
min_confidence: high
include_secret: true
detail: summary
finding_filter: only-active
prevent_empty: true
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) = parse(&["kingfisher", "scan", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(scan_args.alert_min_confidence, ConfidenceLevel::High);
assert!(scan_args.alert_include_secret);
assert_eq!(scan_args.alert_detail, kingfisher::alerts::AlertDetail::Summary);
assert_eq!(
scan_args.alert_finding_filter,
kingfisher::alerts::AlertFindingFilter::OnlyActive
);
assert!(scan_args.alert_prevent_empty);
}
#[test]
fn cli_alert_flag_beats_config_default() {
let yaml = r#"
alerts:
defaults:
min_confidence: high
finding_filter: access-map-only
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) = parse(&[
"kingfisher",
"scan",
"--alert-min-confidence",
"low",
"--alert-finding-filter",
"exclude-inactive",
".",
]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(scan_args.alert_min_confidence, ConfidenceLevel::Low);
assert_eq!(
scan_args.alert_finding_filter,
kingfisher::alerts::AlertFindingFilter::ExcludeInactive
);
}
#[test]
fn config_init_round_trips_supplied_flags_only() {
use kingfisher::cli::config::{ConfigConfidence, ConfigReportFormat, parse_str};
let argv = &[
"kingfisher",
"config",
"init",
"--confidence",
"high",
"--redact",
"--exclude",
"vendor/",
"--skip-word",
"EXAMPLE",
"--exclude-rule",
"betterleaks.github-pat",
"--format",
"toon",
"--alert-min-confidence",
"high",
"--alert-webhook",
"https://hooks.slack.com/services/T0/B0/AAA",
"--tls-mode",
"lax",
];
let matches = CommandLineArgs::command().try_get_matches_from(argv).unwrap();
let parsed = CommandLineArgs::from_arg_matches(&matches).unwrap();
let global_args = parsed.global_args.clone();
let init_matches =
matches.subcommand_matches("config").unwrap().subcommand_matches("init").unwrap();
let scan_args = match parsed.command {
Command::Config(c) => match c.command {
kingfisher::cli::commands::config_command::ConfigSubcommand::Init(args) => {
args.scan_args
}
},
_ => panic!("expected config init"),
};
let yaml = super::build_config_yaml(&scan_args, &global_args, init_matches).unwrap();
let cfg = parse_str(&yaml).expect("emitted YAML must round-trip");
assert!(matches!(cfg.scan.confidence, Some(ConfigConfidence::High)));
assert_eq!(cfg.scan.redact, Some(true));
assert!(cfg.scan.no_dedup.is_none(), "should not emit unset bools");
assert!(cfg.scan.jobs.is_none(), "should not emit clap-default scalars");
assert_eq!(cfg.filters.exclude, vec!["vendor/".to_string()]);
assert_eq!(cfg.filters.skip_words, vec!["EXAMPLE".to_string()]);
assert!(cfg.filters.max_file_size_mb.is_none(), "should not emit unset filters");
assert_eq!(cfg.rules.disabled, vec!["betterleaks.github-pat".to_string()]);
assert!(matches!(cfg.output.format, Some(ConfigReportFormat::Toon)));
assert!(cfg.output.path.is_none());
assert!(matches!(cfg.alerts.defaults.min_confidence, Some(ConfigConfidence::High)));
assert_eq!(cfg.alerts.webhooks.len(), 1);
assert_eq!(cfg.alerts.webhooks[0].url, "https://hooks.slack.com/services/T0/B0/AAA");
assert!(matches!(cfg.global.tls_mode, Some(kingfisher::cli::config::ConfigTlsMode::Lax)));
}
#[test]
fn config_init_preserves_raw_api_url_strings() {
use kingfisher::cli::config::parse_str;
let argv = &[
"kingfisher",
"config",
"init",
"--github-api-url",
"https://ghe.corp.example.com/api/v3",
"--gitlab-api-url",
"https://gitlab.corp.example.com",
];
let matches = CommandLineArgs::command().try_get_matches_from(argv).unwrap();
let parsed = CommandLineArgs::from_arg_matches(&matches).unwrap();
let global_args = parsed.global_args.clone();
let init_matches =
matches.subcommand_matches("config").unwrap().subcommand_matches("init").unwrap();
let scan_args = match parsed.command {
Command::Config(c) => match c.command {
kingfisher::cli::commands::config_command::ConfigSubcommand::Init(args) => {
args.scan_args
}
},
_ => panic!("expected config init"),
};
let yaml = super::build_config_yaml(&scan_args, &global_args, init_matches).unwrap();
let cfg = parse_str(&yaml).expect("emitted YAML must round-trip");
assert_eq!(
cfg.git.github_api_url.as_deref(),
Some("https://ghe.corp.example.com/api/v3"),
"github_api_url must preserve user input verbatim, no trailing-slash rewrite",
);
assert_eq!(
cfg.git.gitlab_api_url.as_deref(),
Some("https://gitlab.corp.example.com"),
"gitlab_api_url must preserve user input verbatim, no trailing-slash rewrite",
);
let argv = &[
"kingfisher",
"config",
"init",
"--github-api-url",
"https://ghe.corp.example.com/api/v3/",
];
let matches = CommandLineArgs::command().try_get_matches_from(argv).unwrap();
let parsed = CommandLineArgs::from_arg_matches(&matches).unwrap();
let global_args = parsed.global_args.clone();
let init_matches =
matches.subcommand_matches("config").unwrap().subcommand_matches("init").unwrap();
let scan_args = match parsed.command {
Command::Config(c) => match c.command {
kingfisher::cli::commands::config_command::ConfigSubcommand::Init(args) => {
args.scan_args
}
},
_ => panic!("expected config init"),
};
let yaml = super::build_config_yaml(&scan_args, &global_args, init_matches).unwrap();
let cfg = parse_str(&yaml).expect("emitted YAML must round-trip");
assert_eq!(
cfg.git.github_api_url.as_deref(),
Some("https://ghe.corp.example.com/api/v3/"),
"github_api_url must preserve a user-supplied trailing slash",
);
}
#[test]
fn config_init_with_no_flags_emits_placeholder_comment() {
let argv = &["kingfisher", "config", "init"];
let matches = CommandLineArgs::command().try_get_matches_from(argv).unwrap();
let parsed = CommandLineArgs::from_arg_matches(&matches).unwrap();
let global_args = parsed.global_args.clone();
let init_matches =
matches.subcommand_matches("config").unwrap().subcommand_matches("init").unwrap();
let scan_args = match parsed.command {
Command::Config(c) => match c.command {
kingfisher::cli::commands::config_command::ConfigSubcommand::Init(args) => {
args.scan_args
}
},
_ => panic!("expected config init"),
};
let yaml = super::build_config_yaml(&scan_args, &global_args, init_matches).unwrap();
assert!(yaml.contains("no flags supplied"), "expected no-op header, got:\n{yaml}");
assert!(!yaml.trim().ends_with("{}"));
}
#[test]
fn global_section_updates_global_args_when_cli_default() {
let yaml = r#"
global:
tls_mode: lax
allow_internal_ips: true
endpoints:
- github=https://ghe.example.com/api/v3/
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) = parse(&["kingfisher", "scan", "."]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(global_args.tls_mode, kingfisher::cli::global::TlsMode::Lax);
assert!(global_args.allow_internal_ips);
assert_eq!(global_args.endpoint.len(), 1);
}
#[test]
fn github_subcommand_api_url_beats_config() {
let yaml = r#"
git:
github_api_url: https://ghe-from-config.example.com/api/v3/
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) = parse(&[
"kingfisher",
"scan",
"github",
"--organization",
"my-org",
"--api-url",
"https://ghe-from-cli.example.com/api/v3/",
]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(
scan_args.input_specifier_args.github_api_url.as_str(),
"https://ghe-from-cli.example.com/api/v3/",
);
}
#[test]
fn github_config_wins_when_subcommand_api_url_default() {
let yaml = r#"
git:
github_api_url: https://ghe-from-config.example.com/api/v3/
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) = parse(&["kingfisher", "scan", "github", "--organization", "my-org"]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(
scan_args.input_specifier_args.github_api_url.as_str(),
"https://ghe-from-config.example.com/api/v3/",
);
}
#[test]
fn gitlab_subcommand_api_url_beats_config() {
let yaml = r#"
git:
gitlab_api_url: https://gitlab-from-config.example.com/
"#;
let cfg = parse_str(yaml).unwrap();
let (args, matches) = parse(&[
"kingfisher",
"scan",
"gitlab",
"--group",
"my-group",
"--api-url",
"https://gitlab-from-cli.example.com/",
]);
let mut global_args = args.global_args.clone();
let mut scan_args = into_scan(args);
super::apply_config(
&mut scan_args,
&mut global_args,
&cfg,
matches.subcommand_matches("scan"),
);
assert_eq!(
scan_args.input_specifier_args.gitlab_api_url.as_str(),
"https://gitlab-from-cli.example.com/",
);
}
}