use super::inference::{surrounding_line_window, COMMENT_MARKERS};
use crate::ascii_ci::ci_find;
use keyhog_core::git_lfs;
use std::collections::BTreeSet;
pub(crate) fn is_false_positive_match_context(
text: &str,
match_start: usize,
file_path: Option<&str>,
) -> bool {
is_false_positive_match_context_with_path(text, match_start, file_path, None)
}
pub(crate) fn is_false_positive_match_context_with_path(
text: &str,
match_start: usize,
_file_path: Option<&str>,
path_lower: Option<&str>,
) -> bool {
let window = surrounding_line_window(text, match_start, 1);
let bytes = window.as_bytes();
let (current_match_line, current_match_offset) = line_at_offset(text, match_start);
let current_line_bytes = current_match_line.as_bytes();
is_go_sum_checksum_bytes(current_line_bytes, path_lower)
|| is_integrity_hash_bytes(current_line_bytes)
|| (git_lfs::is_git_lfs_oid_line(current_line_bytes) && git_lfs::is_git_lfs_pointer(bytes))
|| is_renovate_digest_match_context(current_match_line.as_bytes(), current_match_offset)
|| is_cors_header_bytes(current_line_bytes)
|| is_http_cache_header_bytes(current_line_bytes)
|| has_disclaimer_comment_bytes(current_line_bytes)
|| is_public_pem_block_at(text, match_start)
}
pub(crate) fn is_public_pem_block_at(text: &str, match_start: usize) -> bool {
const BEGIN: &[u8] = b"-----BEGIN ";
const END: &[u8] = b"-----END ";
const TRAILER: &[u8] = b"-----";
let bytes = text.as_bytes();
let match_start = match_start.min(bytes.len());
let Some(begin) = memchr::memmem::rfind(&bytes[..match_start], BEGIN) else {
return false;
};
let header_tail = &bytes[begin..];
let header_end =
memchr::memchr(b'\n', header_tail).map_or(header_tail.len(), |newline| newline);
let header = trim_optional_carriage_return(&header_tail[..header_end]);
let Some(label) = pem_label(header, BEGIN, TRAILER) else {
return false;
};
if !matches!(
label,
b"CERTIFICATE"
| b"TRUSTED CERTIFICATE"
| b"X509 CERTIFICATE"
| b"PUBLIC KEY"
| b"RSA PUBLIC KEY"
| b"SSH2 PUBLIC KEY"
) {
return false;
}
let body_before_match = &bytes[begin + header_end..match_start];
if memchr::memmem::find(body_before_match, END).is_some() {
return false;
}
let after_match = &bytes[match_start..];
let Some(end_offset) = memchr::memmem::find(after_match, END) else {
return false;
};
if memchr::memmem::find(&after_match[..end_offset], BEGIN).is_some() {
return false;
}
let end_tail = &after_match[end_offset..];
let end_line_end = memchr::memchr(b'\n', end_tail).map_or(end_tail.len(), |newline| newline);
let end_header = trim_optional_carriage_return(&end_tail[..end_line_end]);
pem_label(end_header, END, TRAILER).is_some_and(|end_label| end_label == label)
}
fn pem_label<'a>(header: &'a [u8], prefix: &[u8], trailer: &[u8]) -> Option<&'a [u8]> {
header.strip_prefix(prefix)?.strip_suffix(trailer)
}
fn trim_optional_carriage_return(line: &[u8]) -> &[u8] {
if line.ends_with(b"\r") {
&line[..line.len() - 1]
} else {
line
}
}
static DISCLAIMER_PHRASES: std::sync::LazyLock<Vec<String>> = std::sync::LazyLock::new(|| {
match parse_disclaimer_phrases(include_str!("../../data/disclaimer-phrases.toml")) {
Ok(phrases) => phrases,
Err(error) => {
panic!(
"crates/scanner/data/disclaimer-phrases.toml is invalid: {error}. \
Fix the bundled Tier-B disclaimer phrases; refusing to run without \
disclaimer suppression truth."
)
}
}
});
#[derive(serde::Deserialize)]
struct DisclaimerFile {
schema_version: u32,
phrases: Vec<String>,
}
pub(crate) fn parse_disclaimer_phrases(raw: &str) -> Result<Vec<String>, String> {
let parsed: DisclaimerFile =
toml::from_str(raw).map_err(|error| format!("invalid disclaimer-phrases.toml: {error}"))?;
if parsed.schema_version != 1 {
return Err(format!(
"unsupported disclaimer phrase schema_version {}",
parsed.schema_version
));
}
let mut seen = BTreeSet::new();
let mut phrases = Vec::with_capacity(parsed.phrases.len());
for raw_phrase in parsed.phrases {
let phrase = raw_phrase.trim();
if phrase.is_empty() {
return Err("disclaimer phrase entries must not be empty".to_string());
}
if !phrase.is_ascii() || phrase != phrase.to_ascii_lowercase() {
return Err(format!(
"disclaimer phrase {phrase:?} must be lowercase ASCII"
));
}
if !seen.insert(phrase.to_string()) {
return Err(format!("duplicate disclaimer phrase {phrase:?}"));
}
phrases.push(phrase.to_string());
}
if phrases.is_empty() {
return Err("disclaimer phrases must contain at least one entry".to_string());
}
Ok(phrases)
}
pub(crate) fn has_disclaimer_comment_bytes(bytes: &[u8]) -> bool {
let phrases: &[String] = &DISCLAIMER_PHRASES;
for marker in COMMENT_MARKERS.iter().map(|m| m.as_bytes()) {
let m_len = marker.len();
let first_lower = marker[0];
let first_upper = first_lower.to_ascii_uppercase();
let mut q_pos = 0usize;
let mut quote: Option<u8> = None;
let mut escaped = false;
for start in memchr::memchr2_iter(first_lower, first_upper, bytes) {
if start + m_len > bytes.len() {
break;
}
if !bytes[start..start + m_len].eq_ignore_ascii_case(marker) {
continue;
}
if marker == b"--" && bytes.get(start + m_len).copied() == Some(b'-') {
continue;
}
if !comment_marker_has_boundary(bytes, start) {
continue;
}
while q_pos < start {
let byte = bytes[q_pos];
if escaped {
escaped = false;
} else if byte == b'\\' {
escaped = true;
} else {
match quote {
Some(open) if byte == open => quote = None,
None if byte == b'"' || byte == b'\'' => quote = Some(byte),
_ => {}
}
}
q_pos += 1;
}
if quote.is_some() {
continue;
}
let tail_start = start + m_len;
let line_end = memchr::memchr(b'\n', &bytes[tail_start..])
.map_or(bytes.len(), |nl| tail_start + nl);
let comment_tail = &bytes[tail_start..line_end];
for phrase in phrases {
if ci_find(comment_tail, phrase.as_bytes()) {
return true;
}
}
break;
}
}
false
}
fn comment_marker_has_boundary(bytes: &[u8], start: usize) -> bool {
start == 0 || bytes[start - 1].is_ascii_whitespace()
}
pub(crate) fn is_false_positive_context(
lines: &[&str],
line_idx: usize,
file_path: Option<&str>,
) -> bool {
if line_idx >= lines.len() {
return false;
}
let line_bytes = lines[line_idx].as_bytes();
is_go_sum_checksum_bytes(line_bytes, file_path)
|| is_integrity_hash_bytes(line_bytes)
|| is_configmap_binary_data_context(lines, line_idx, line_bytes)
|| is_git_lfs_pointer_context_with_lines(lines, line_idx, line_bytes)
|| is_cors_header_bytes(line_bytes)
|| is_http_cache_header_bytes(line_bytes)
}
fn is_go_sum_checksum_bytes(bytes: &[u8], path: Option<&str>) -> bool {
let path_is_go_sum =
path.is_some_and(|p| crate::ascii_ci::ends_with_ignore_ascii_case(p.as_bytes(), b"go.sum"));
let mut start = 0;
while let Some(h1_pos) = find_h1_marker(bytes, start) {
if has_h1_token_boundary(bytes, h1_pos)
&& (path_is_go_sum || has_strict_go_sum_checksum_shape(bytes, h1_pos))
{
return true;
}
start = h1_pos + b"h1:".len();
}
false
}
fn find_h1_marker(bytes: &[u8], start: usize) -> Option<usize> {
let mut cursor = start;
while let Some(colon_rel) = memchr::memchr(b':', bytes.get(cursor..)?) {
let colon = cursor + colon_rel;
if colon >= 2 && bytes[colon - 1] == b'1' && bytes[colon - 2].eq_ignore_ascii_case(&b'h') {
return Some(colon - 2);
}
cursor = colon + 1;
}
None
}
fn has_h1_token_boundary(bytes: &[u8], h1_pos: usize) -> bool {
h1_pos == 0 || bytes[h1_pos - 1].is_ascii_whitespace()
}
const GO_SUM_H1_BASE64_DIGEST_LEN: usize = 44;
fn has_strict_go_sum_checksum_shape(bytes: &[u8], h1_pos: usize) -> bool {
if count_ascii_fields(&bytes[..h1_pos]) < 2 {
return false;
}
let digest_start = h1_pos + b"h1:".len();
let Some(digest) = bytes.get(digest_start..digest_start + GO_SUM_H1_BASE64_DIGEST_LEN) else {
return false;
};
digest
.iter()
.all(|&byte| crate::decode::is_standard_base64_byte(byte))
&& bytes
.get(digest_start + GO_SUM_H1_BASE64_DIGEST_LEN)
.is_none_or(u8::is_ascii_whitespace)
}
fn count_ascii_fields(bytes: &[u8]) -> usize {
let mut count = 0;
let mut in_field = false;
for byte in bytes {
if byte.is_ascii_whitespace() {
in_field = false;
} else if !in_field {
count += 1;
in_field = true;
}
}
count
}
pub(crate) fn is_integrity_hash_bytes(bytes: &[u8]) -> bool {
ci_find(bytes, b"integrity")
&& crate::suppression::shape::HASH_ALGO_INTEGRITY_LABELS
.iter()
.any(|label| contains_sri_hash_value(bytes, label.as_bytes()))
}
fn contains_sri_hash_value(bytes: &[u8], prefix: &[u8]) -> bool {
for start in crate::ascii_ci::ci_find_iter(bytes, prefix) {
let value_start = start + prefix.len();
let mut value_end = value_start;
while let Some(byte) = bytes.get(value_end) {
if crate::decode::is_standard_base64_byte(*byte) {
value_end += 1;
} else {
break;
}
}
if is_base64_scalar(&bytes[value_start..value_end]) {
return true;
}
}
false
}
fn is_configmap_binary_data_context(lines: &[&str], line_idx: usize, line_bytes: &[u8]) -> bool {
is_configmap_binary_data_value_line(line_bytes)
&& is_inside_configmap_binary_data_block(lines, line_idx)
}
const BLOCK_HEADER_LOOKBACK: usize = 4096;
fn is_inside_configmap_binary_data_block(lines: &[&str], line_idx: usize) -> bool {
let Some(current_line) = lines.get(line_idx) else {
return false;
};
let current_indent = leading_ascii_space_count(current_line.as_bytes());
if current_indent == 0 {
return false;
}
let start = line_idx.saturating_sub(BLOCK_HEADER_LOOKBACK);
for candidate in lines[start..line_idx].iter().rev() {
let bytes = candidate.as_bytes();
let trimmed = trim_ascii_bytes(bytes);
if trimmed.is_empty() {
continue;
}
let indent = leading_ascii_space_count(bytes);
if indent >= current_indent {
continue;
}
return trimmed.eq_ignore_ascii_case(b"binarydata:");
}
false
}
fn leading_ascii_space_count(bytes: &[u8]) -> usize {
bytes.iter().take_while(|byte| **byte == b' ').count()
}
fn is_configmap_binary_data_value_line(bytes: &[u8]) -> bool {
let trimmed = trim_ascii_bytes(bytes);
let Some(colon) = memchr::memchr(b':', trimmed) else {
return false;
};
let key = trim_ascii_bytes(&trimmed[..colon]);
let value = trim_ascii_bytes(&trimmed[colon + 1..]);
!key.is_empty()
&& is_yaml_scalar_key(key)
&& is_base64_scalar(strip_balanced_ascii_quotes(value))
}
fn is_yaml_scalar_key(bytes: &[u8]) -> bool {
bytes
.iter()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
}
fn strip_balanced_ascii_quotes(bytes: &[u8]) -> &[u8] {
if bytes.len() >= 2
&& ((bytes[0] == b'"' && bytes[bytes.len() - 1] == b'"')
|| (bytes[0] == b'\'' && bytes[bytes.len() - 1] == b'\''))
{
&bytes[1..bytes.len() - 1]
} else {
bytes
}
}
fn is_base64_scalar(bytes: &[u8]) -> bool {
bytes.len() >= 8
&& bytes.len() % 4 == 0
&& bytes
.iter()
.all(|&byte| crate::decode::is_standard_base64_byte(byte))
}
const GIT_LFS_POINTER_LOOKAROUND_LINES: usize = 3;
fn is_git_lfs_pointer_context_with_lines(
lines: &[&str],
line_idx: usize,
line_bytes: &[u8],
) -> bool {
git_lfs::is_git_lfs_oid_line(line_bytes)
&& nearby_lines_contain(
lines,
line_idx,
GIT_LFS_POINTER_LOOKAROUND_LINES,
|candidate| git_lfs::is_git_lfs_version_line(candidate.as_bytes()),
)
&& following_lines_contain(
lines,
line_idx,
GIT_LFS_POINTER_LOOKAROUND_LINES,
|candidate| git_lfs::is_git_lfs_size_line(candidate.as_bytes()),
)
}
fn trim_ascii_bytes(bytes: &[u8]) -> &[u8] {
let start = bytes
.iter()
.position(|byte| !byte.is_ascii_whitespace())
.unwrap_or(bytes.len()); let end = bytes
.iter()
.rposition(|byte| !byte.is_ascii_whitespace())
.map_or(start, |idx| idx + 1);
&bytes[start..end]
}
fn is_renovate_digest_match_context(bytes: &[u8], match_offset: usize) -> bool {
let match_offset = match_offset.min(bytes.len());
let mut cursor = 0;
while cursor < bytes.len() {
let Some(relative_start) = crate::ascii_ci::ci_find_at(&bytes[cursor..], b"renovate/")
else {
return false;
};
let start = cursor + relative_start;
let branch_start = start + b"renovate/".len();
let mut branch_end = branch_start;
while let Some(byte) = bytes.get(branch_end) {
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.') {
branch_end += 1;
} else {
break;
}
}
if branch_end > branch_start
&& match_offset >= branch_start
&& match_offset < branch_end
&& hex_run_contains_offset(bytes, branch_start, branch_end, match_offset)
{
return true;
}
cursor = branch_end.max(start + 1);
}
false
}
fn hex_run_contains_offset(bytes: &[u8], start: usize, end: usize, offset: usize) -> bool {
let mut run_start = start;
let mut run_len = 0usize;
for idx in start..end {
if bytes[idx].is_ascii_hexdigit() {
if run_len == 0 {
run_start = idx;
}
run_len += 1;
continue;
}
if run_len >= 8 && offset >= run_start && offset < idx {
return true;
}
run_len = 0;
}
run_len >= 8 && offset >= run_start && offset < end
}
fn line_at_offset(text: &str, offset: usize) -> (&str, usize) {
let bytes = text.as_bytes();
let safe_offset = offset.min(bytes.len());
let line_start = bytes[..safe_offset]
.iter()
.rposition(|byte| *byte == b'\n')
.map_or(0, |idx| idx + 1);
let line_end = bytes[safe_offset..]
.iter()
.position(|byte| *byte == b'\n')
.map_or(bytes.len(), |idx| safe_offset + idx);
let line_start = crate::engine::ceil_char_boundary(text, line_start);
let mut line_end = crate::engine::floor_char_boundary(text, line_end);
if line_end < line_start {
line_end = line_start;
}
let relative = safe_offset
.saturating_sub(line_start)
.min(line_end - line_start);
(&text[line_start..line_end], relative)
}
#[derive(serde::Deserialize)]
struct FalsePositiveMarkers {
cors_headers: Vec<String>,
}
fn parse_false_positive_markers(raw: &str) -> Result<Vec<String>, String> {
toml::from_str::<FalsePositiveMarkers>(raw)
.map(|parsed| parsed.cors_headers)
.map_err(|error| error.to_string())
}
static CORS_HEADERS: std::sync::LazyLock<Vec<String>> = std::sync::LazyLock::new(|| {
match parse_false_positive_markers(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/rules/false-positive-markers.toml"
))) {
Ok(cors_headers) => cors_headers,
Err(error) => panic!(
"rules/false-positive-markers.toml is invalid: {error}. \
Fix the bundled Tier-B false-positive markers list."
),
}
});
fn is_cors_header_bytes(bytes: &[u8]) -> bool {
let allowed: &[String] = &CORS_HEADERS;
header_name_matches(bytes, allowed)
}
fn is_http_cache_header_bytes(bytes: &[u8]) -> bool {
let allowed: &[&[u8]] = &[b"etag"];
header_name_matches(bytes, allowed)
}
fn header_name_matches<T: AsRef<[u8]>>(bytes: &[u8], allowed: &[T]) -> bool {
let trimmed = trim_ascii_bytes(bytes);
let Some(colon) = memchr::memchr(b':', trimmed) else {
return false;
};
let name = trim_ascii_bytes(&trimmed[..colon]);
if name.is_empty() {
return false;
}
if !name
.iter()
.all(|byte| byte.is_ascii_alphanumeric() || *byte == b'-')
{
return false;
}
allowed
.iter()
.any(|candidate| name.eq_ignore_ascii_case(candidate.as_ref()))
}
fn nearby_lines_contain(
lines: &[&str],
line_idx: usize,
lookback_lines: usize,
predicate: impl Fn(&str) -> bool,
) -> bool {
let end = (line_idx + 1).min(lines.len());
let start = line_idx.saturating_sub(lookback_lines).min(end);
lines[start..end].iter().copied().any(predicate)
}
fn following_lines_contain(
lines: &[&str],
line_idx: usize,
lookahead_lines: usize,
predicate: impl Fn(&str) -> bool,
) -> bool {
let start = line_idx.saturating_add(1);
let end = line_idx
.saturating_add(lookahead_lines)
.saturating_add(1)
.min(lines.len());
start < end && lines[start..end].iter().copied().any(predicate)
}