pub const UTF8_BOM: &[u8; 3] = b"\xEF\xBB\xBF";
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct DecodedText<'a> {
pub text: &'a str,
pub leading_bytes: u32,
}
pub fn decode_searchable_text(bytes: &[u8]) -> Result<DecodedText<'_>, std::str::Utf8Error> {
let (text_bytes, leading_bytes) = bytes
.strip_prefix(UTF8_BOM)
.map_or((bytes, 0), |without_bom| {
(without_bom, UTF8_BOM.len() as u32)
});
std::str::from_utf8(text_bytes).map(|text| DecodedText {
text,
leading_bytes,
})
}
pub const SECRET_POLICY_VERSION: u32 = 5;
pub const SECRET_SCAN_MAX_BYTES: usize = crate::domain::index::METADATA_ONLY_CODE_BYTES as usize;
pub const INDETERMINATE_RULE_ID: &str = "secret.detector.indeterminate";
const CONTEXT_ASSIGNMENT_RULE_ID: &str = "secret.context-assignment";
const URI_CREDENTIALS_RULE_ID: &str = "secret.uri-credentials";
const CONTEXT_ASSIGNMENT_MIN_PAYLOAD: usize = 8;
const CONTEXT_ASSIGNMENT_SCAN_BOUND: usize = 512;
const CHAR_LITERAL_MAX_CONTENT: usize = 3;
pub fn exceeds_scan_limit(len: usize) -> bool {
len > SECRET_SCAN_MAX_BYTES
}
pub const FINDING_LINE_RANGES_KEPT: usize = 11;
fn first_line_ranges(line_bits: &[u64]) -> Vec<(u32, u32)> {
let mut ranges: Vec<(u32, u32)> = Vec::new();
for (word_index, &word) in line_bits.iter().enumerate() {
let mut rest = word;
while rest != 0 {
let bit = word_index * 64 + rest.trailing_zeros() as usize;
rest &= rest - 1;
let line = u32::try_from(bit + 1).unwrap_or(u32::MAX);
match ranges.last_mut() {
Some((_, end)) if end.checked_add(1) == Some(line) => *end = line,
_ => {
if ranges.len() == FINDING_LINE_RANGES_KEPT {
return ranges;
}
ranges.push((line, line));
}
}
}
}
ranges
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum DetectorFailure {
PolicyCompilation,
ResourceLimit,
Internal,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum SecretScan {
Clean,
Sensitive {
rule_ids: Vec<&'static str>,
finding_count: u32,
line_ranges: Vec<(u32, u32)>,
},
Indeterminate {
reason: DetectorFailure,
},
}
struct SecretRule {
id: &'static str,
keywords: &'static [&'static [u8]],
pattern: regex::bytes::Regex,
secret_capture: usize,
placeholders_allowed: bool,
}
static SECRET_RULES: std::sync::OnceLock<Result<Vec<SecretRule>, DetectorFailure>> =
std::sync::OnceLock::new();
fn compile_secret_rules() -> Result<Vec<SecretRule>, DetectorFailure> {
#[allow(clippy::type_complexity)]
let definitions: &[(&str, &[&[u8]], &str, usize, bool)] = &[
(
"secret.private-key-envelope",
&[b"PRIVATE KEY"],
r"-----BEGIN (?:RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----",
0,
false,
),
(
"secret.authorization-header",
&[b"authorization"],
r"(?i)authorization[ \t]*[:=][ \t]*(?:bearer|basic)[ \t]+([A-Za-z0-9._~+/=-]{8,})",
1,
false,
),
(
"secret.provider-token",
&[b"gh"],
r"gh[pousr]_[A-Za-z0-9]{36,}",
0,
false,
),
(
CONTEXT_ASSIGNMENT_RULE_ID,
&[b"key", b"secret", b"token", b"password", b"passwd", b"pwd"],
r#"(?i)(?:api[_-]?key|secret(?:[_-]?access[_-]?key)?|token|password|passwd|pwd|client[_-]?secret)[ \t]*[:=][ \t]*(?:\[[ \t]*)?(?:(?:["'][^"'\n]{0,7}["'][ \t]*,[ \t]*)+["']|["']?)([^\s"'#]{8,})"#,
1,
true,
),
(
URI_CREDENTIALS_RULE_ID,
&[b"://"],
r"://[^/\s:@]+:([^@\s/]{4,})@",
1,
false,
),
];
definitions
.iter()
.map(
|(id, keywords, pattern, secret_capture, placeholders_allowed)| {
regex::bytes::Regex::new(pattern)
.map(|pattern| SecretRule {
id,
keywords,
pattern,
secret_capture: *secret_capture,
placeholders_allowed: *placeholders_allowed,
})
.map_err(|_| DetectorFailure::PolicyCompilation)
},
)
.collect()
}
fn contains_ascii_case_insensitive(haystack: &[u8], needle: &[u8]) -> bool {
!needle.is_empty()
&& haystack
.windows(needle.len())
.any(|window| window.eq_ignore_ascii_case(needle))
}
fn is_placeholder(value: &[u8]) -> bool {
let Ok(value) = std::str::from_utf8(value) else {
return false;
};
let normalized = value
.trim_matches(|character: char| {
matches!(character, '"' | '\'' | '`' | '<' | '>' | '[' | ']')
})
.to_ascii_lowercase();
matches!(
normalized.as_str(),
"example"
| "sample"
| "placeholder"
| "changeme"
| "change-me"
| "change_me"
| "redacted"
| "replace-me"
| "replace_me"
| "dummy"
| "fake"
| "test-value"
) || normalized.starts_with("your_")
|| normalized.starts_with("your-")
|| is_placeholder_only_expression(&normalized)
}
fn is_placeholder_only_expression(value: &str) -> bool {
const GROUPS: [(&str, &str); 3] = [("${{", "}}"), ("{{", "}}"), ("${", "}")];
let mut rest = value;
let mut matched_any = false;
'consume: while !rest.is_empty() {
for (open, close) in GROUPS {
let Some(interior) = rest.strip_prefix(open) else {
continue;
};
let Some(end) = interior.find(close) else {
continue;
};
if end == 0
|| interior[..end]
.bytes()
.any(|byte| matches!(byte, b'{' | b'}'))
{
continue;
}
rest = &interior[end + close.len()..];
matched_any = true;
continue 'consume;
}
return false;
}
matched_any
}
fn right_hand_side_continuation(
bytes: &[u8],
capture_start: usize,
capture_end: usize,
) -> Option<usize> {
let opener = capture_start
.checked_sub(1)
.and_then(|index| bytes.get(index).copied())
.filter(|byte| matches!(byte, b'"' | b'\'' | b'`'));
match (opener, bytes.get(capture_end)) {
(_, Some(b'"' | b'\'' | b'`')) => Some(capture_end + 1),
(Some(quote), _) => bytes[capture_end..]
.iter()
.take_while(|byte| **byte != b'\n')
.position(|byte| *byte == quote)
.map(|offset| capture_end + offset + 1),
(None, _) => Some(capture_end),
}
}
fn is_angle_placeholder(bytes: &[u8], start: usize) -> bool {
if bytes.get(start) != Some(&b'<') {
return false;
}
let line_end = bytes[start..]
.iter()
.position(|byte| *byte == b'\n')
.map_or(bytes.len(), |offset| start + offset);
let opener = start
.checked_sub(1)
.and_then(|index| bytes.get(index).copied())
.filter(|byte| matches!(byte, b'"' | b'\'' | b'`'));
let end = match opener {
Some(quote) => match bytes[start..line_end]
.iter()
.position(|byte| *byte == quote)
{
Some(offset) => start + offset,
None => return false,
},
None => line_end,
};
let value = bytes[start..end].trim_ascii_end();
let Some(first_close) = value.iter().position(|byte| *byte == b'>') else {
return false;
};
if !value[first_close..].iter().all(|byte| *byte == b'>') {
return false;
}
let interior = &value[..first_close];
let interior = &interior[interior.iter().take_while(|byte| **byte == b'<').count()..];
interior.iter().any(u8::is_ascii_whitespace)
|| !interior.iter().any(u8::is_ascii_digit)
|| is_placeholder(interior)
}
fn uri_credentials_are_placeholder(
bytes: &[u8],
userinfo_start: usize,
userinfo_end: usize,
) -> bool {
let userinfo = &bytes[userinfo_start..userinfo_end];
let placeholder_userinfo = is_placeholder(userinfo)
|| matches!(
userinfo.to_ascii_lowercase().as_slice(),
b"pass" | b"password" | b"pw" | b"secret"
)
|| userinfo.iter().all(|byte| byte.eq_ignore_ascii_case(&b'x'));
let host = bytes[userinfo_end + 1..]
.iter()
.take_while(|byte| {
!matches!(
byte,
b'/' | b':' | b'?' | b'#' | b'"' | b'\'' | b'`' | b')' | b']' | b',' | b';'
) && !byte.is_ascii_whitespace()
})
.count();
let host = std::str::from_utf8(&bytes[userinfo_end + 1..userinfo_end + 1 + host])
.unwrap_or("")
.to_ascii_lowercase();
let placeholder_host = matches!(host.as_str(), "host" | "localhost")
|| host.starts_with("example.")
|| (host.len() > 2 && host.starts_with('<') && host.ends_with('>'))
|| is_placeholder_only_expression(&host);
placeholder_userinfo && placeholder_host
}
fn assignment_is_code_expression(
path: &str,
bytes: &[u8],
match_start: usize,
value_start: usize,
value: &[u8],
) -> bool {
let Some(language) =
crate::domain::LanguageId::from_path(path).filter(|language| language.is_code_language())
else {
return false;
};
if separator_is_path_not_assignment(bytes, match_start, value_start) {
return true;
}
let rust = language == crate::domain::LanguageId::Rust;
let quote_before_value = value_start
.checked_sub(1)
.and_then(|index| bytes.get(index).copied())
.is_some_and(|byte| matches!(byte, b'"' | b'\''));
if quote_before_value {
let quote = bytes[value_start - 1];
let compact_keyword_literal = bytes[..value_start - 1]
.iter()
.rposition(|byte| *byte == quote || byte.is_ascii_whitespace())
.is_some_and(|open| bytes[open] == quote && open < match_start);
let closes_keyword_literal = compact_keyword_literal
&& match_is_inside_string_literal(bytes, match_start, rust)
&& !match_is_inside_string_literal(bytes, value_start, rust);
return closes_keyword_literal
&& !expression_carries_quoted_payload(bytes, value_start, true);
}
if match_is_inside_string_literal(bytes, value_start, rust) {
return capture_is_single_interpolation(value);
}
!expression_carries_quoted_payload(bytes, value_start, false)
}
fn match_is_inside_string_literal(bytes: &[u8], value_start: usize, rust: bool) -> bool {
let line_start = bytes[..value_start]
.iter()
.rposition(|byte| *byte == b'\n')
.map_or(0, |index| index + 1);
let (mut double, mut single, mut backtick) = (false, false, false);
let mut index = line_start;
while index < value_start {
match bytes[index] {
b'\\' => {
index += 2;
continue;
}
b'"' => double = !double,
b'\''
if rust
&& bytes
.get(index + 1)
.is_some_and(|next| next.is_ascii_alphabetic() || *next == b'_')
&& bytes.get(index + 2) != Some(&b'\'') => {}
b'\'' => single = !single,
b'`' => backtick = !backtick,
_ => {}
}
index += 1;
}
double || single || backtick
}
fn separator_is_path_not_assignment(bytes: &[u8], match_start: usize, value_start: usize) -> bool {
let Some(separator) = bytes[match_start..value_start]
.iter()
.position(|byte| matches!(byte, b':' | b'='))
.map(|offset| match_start + offset)
else {
return false;
};
if bytes[separator] != b':' || bytes.get(separator + 1) != Some(&b':') {
return false;
}
let line_end = bytes[separator..]
.iter()
.position(|byte| *byte == b'\n')
.map_or(bytes.len(), |offset| separator + offset);
if line_end < bytes.len() && line_break_continues_expression(bytes, line_end) {
return false;
}
let rest = &bytes[separator + 2..line_end];
let mut depth = 0_u32;
for byte in rest {
match byte {
b'(' | b'[' | b'{' => depth += 1,
b')' | b']' | b'}' => depth = depth.saturating_sub(1),
_ => {}
}
}
let fences_payload = rest.iter().enumerate().any(|(index, quote)| {
matches!(quote, b'"' | b'\'' | b'`') && {
let run = rest[index + 1..]
.iter()
.take_while(|byte| {
!matches!(byte, b'"' | b'\'' | b'`' | b'#') && !byte.is_ascii_whitespace()
})
.count();
run >= CONTEXT_ASSIGNMENT_MIN_PAYLOAD && rest.get(index + 1 + run) == Some(quote)
}
});
depth == 0 && !fences_payload
}
fn capture_is_single_interpolation(value: &[u8]) -> bool {
value.len() >= 3
&& value.first() == Some(&b'{')
&& value.last() == Some(&b'}')
&& !value[1..value.len() - 1]
.iter()
.any(|byte| matches!(byte, b'{' | b'}'))
}
fn line_break_continues_expression(window: &[u8], newline: usize) -> bool {
const CONTINUATION: &[u8] = b"+-*|&^%=.?:\\,";
let trailing = window[..newline]
.iter()
.rposition(|byte| !byte.is_ascii_whitespace())
.map(|index| window[index]);
let next_line = window[newline + 1..].trim_ascii_start();
if trailing == Some(b',') && (next_line.starts_with(b"//") || next_line.starts_with(b"#[")) {
return false;
}
let leading = next_line.first().copied();
trailing.is_some_and(|byte| CONTINUATION.contains(&byte))
|| leading.is_some_and(|byte| CONTINUATION.contains(&byte))
}
fn bounded_char_literal_len(window: &[u8], at: usize) -> Option<usize> {
let last_close = at + 1 + CHAR_LITERAL_MAX_CONTENT;
let mut cursor = at + 1;
let mut carries_bracket = false;
while cursor <= last_close {
match window.get(cursor)? {
b'\\' => cursor += 2,
b'\n' => return None,
b'\'' if cursor > at + 1 => {
return carries_bracket.then_some(cursor + 1 - at);
}
byte => {
carries_bracket |= matches!(byte, b'(' | b')' | b'[' | b']' | b'{' | b'}');
cursor += 1;
}
}
}
None
}
fn expression_carries_quoted_payload(
bytes: &[u8],
from: usize,
after_closed_literal: bool,
) -> bool {
let end = from
.saturating_add(CONTEXT_ASSIGNMENT_SCAN_BOUND)
.min(bytes.len());
let window = &bytes[from..end];
let truncated = end < bytes.len();
let is_payload =
|byte: u8| !matches!(byte, b'"' | b'\'' | b'`' | b'#') && !byte.is_ascii_whitespace();
let mut depth: i32 = 0;
let mut index = 0;
while index < window.len() {
let byte = window[index];
match byte {
b'"' | b'\'' | b'`' => {
let mut run = index + 1;
while run < window.len() && is_payload(window[run]) {
run += 1;
}
if run - (index + 1) >= CONTEXT_ASSIGNMENT_MIN_PAYLOAD
&& window.get(run) == Some(&byte)
{
return true;
}
if byte == b'\'' {
index += bounded_char_literal_len(window, index).unwrap_or(1);
} else {
let mut cursor = index + 1;
loop {
match window.get(cursor) {
None => return true,
Some(b'\\') => cursor += 2,
Some(other) if *other == byte => break,
Some(_) => cursor += 1,
}
}
index = cursor + 1;
}
}
b'(' | b'[' | b'{' => {
depth += 1;
index += 1;
}
b')' | b']' | b'}' => {
depth -= 1;
if depth < 0 {
if !after_closed_literal {
return false;
}
depth = 0;
}
index += 1;
}
b'\n' if depth == 0 => {
if !line_break_continues_expression(window, index) {
return false;
}
index += 1;
}
_ => index += 1,
}
}
truncated || depth != 0
}
pub fn scan_secret_bytes(path: &str, bytes: &[u8]) -> SecretScan {
if exceeds_scan_limit(bytes.len()) {
return SecretScan::Indeterminate {
reason: DetectorFailure::ResourceLimit,
};
}
let rules = match SECRET_RULES.get_or_init(compile_secret_rules) {
Ok(rules) => rules,
Err(reason) => return SecretScan::Indeterminate { reason: *reason },
};
let mut rule_ids = Vec::new();
let mut finding_count = 0_u32;
let mut line_bits: Vec<u64> = Vec::new();
for rule in rules {
if !rule
.keywords
.iter()
.any(|keyword| contains_ascii_case_insensitive(bytes, keyword))
{
continue;
}
let (mut cursor, mut line) = (0_usize, 0_usize);
for captures in rule.pattern.captures_iter(bytes) {
let Some(secret) = captures.get(rule.secret_capture) else {
return SecretScan::Indeterminate {
reason: DetectorFailure::Internal,
};
};
if rule.placeholders_allowed
&& (is_placeholder(secret.as_bytes())
|| is_angle_placeholder(bytes, secret.start()))
{
if right_hand_side_continuation(bytes, secret.start(), secret.end())
.is_some_and(|from| !expression_carries_quoted_payload(bytes, from, false))
{
continue;
}
} else if (rule.id == CONTEXT_ASSIGNMENT_RULE_ID
&& assignment_is_code_expression(
path,
bytes,
captures
.get(0)
.map_or(secret.start(), |whole| whole.start()),
secret.start(),
secret.as_bytes(),
))
|| (rule.id == URI_CREDENTIALS_RULE_ID
&& uri_credentials_are_placeholder(bytes, secret.start(), secret.end()))
{
continue;
}
finding_count = finding_count.saturating_add(1);
if !rule_ids.contains(&rule.id) {
rule_ids.push(rule.id);
}
line += bytes[cursor..secret.start()]
.iter()
.filter(|byte| **byte == b'\n')
.count();
cursor = secret.start();
let word = line / 64;
if word >= line_bits.len() {
line_bits.resize(word + 1, 0);
}
line_bits[word] |= 1 << (line % 64);
}
}
if finding_count == 0 {
SecretScan::Clean
} else {
SecretScan::Sensitive {
rule_ids,
finding_count,
line_ranges: first_line_ranges(&line_bits),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LfsPointerMetadata {
pub declared_oid: Option<String>,
pub declared_size: Option<u64>,
}
pub fn detect_lfs_pointer(bytes: &[u8]) -> Option<LfsPointerMetadata> {
if bytes.len() > 1024 {
return None;
}
let text = std::str::from_utf8(bytes).ok()?;
let lines = text
.lines()
.map(|line| line.strip_suffix('\r').unwrap_or(line))
.collect::<Vec<_>>();
if lines.len() != 3 || lines[0] != "version https://git-lfs.github.com/spec/v1" {
return None;
}
let oid = lines[1].strip_prefix("oid sha256:")?;
if oid.len() != 64 || !oid.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return None;
}
let declared_size = lines[2].strip_prefix("size ")?.parse::<u64>().ok()?;
Some(LfsPointerMetadata {
declared_oid: Some(format!("sha256:{oid}")),
declared_size: Some(declared_size),
})
}
fn is_safe_template_basename(basename: &str) -> bool {
["example", "sample", "template", "dist"]
.iter()
.any(|marker| {
basename.ends_with(&format!(".{marker}")) || basename.starts_with(&format!("{marker}."))
})
}
pub fn sensitive_path_rule(relative_path: &str) -> Option<&'static str> {
let normalized = crate::paths::fold_case(
&relative_path
.split(['/', '\\'])
.filter(|segment| !segment.is_empty() && *segment != ".")
.collect::<Vec<_>>()
.join("/"),
);
let basename = normalized.rsplit('/').next().unwrap_or(normalized.as_str());
if !is_safe_template_basename(basename)
&& (basename == ".env" || basename.starts_with(".env.") || basename.ends_with(".env"))
{
return Some("path.environment-credentials");
}
if basename == ".git-credentials" || matches!(basename, ".netrc" | "_netrc") {
return Some("path.network-credential-store");
}
if basename.ends_with(".key")
|| matches!(basename, "id_rsa" | "id_dsa" | "id_ecdsa" | "id_ed25519")
|| (basename.ends_with(".pem") && basename.contains("private"))
{
return Some("path.private-key-material");
}
if normalized.ends_with("/.aws/credentials")
|| normalized == ".aws/credentials"
|| normalized.ends_with("/.kube/config")
|| normalized == ".kube/config"
|| normalized.ends_with("application_default_credentials.json")
{
return Some("path.cloud-credential-store");
}
if basename.ends_with(".tfstate") || basename.ends_with(".tfstate.backup") {
return Some("path.infrastructure-state");
}
None
}
pub fn sensitive_path_rule_at(
relative_path: &str,
absolute_path: &std::path::Path,
) -> Option<&'static str> {
sensitive_path_rule(relative_path)
.or_else(|| sensitive_path_rule(&absolute_path.to_string_lossy()))
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum StableContentAdmission {
Admitted,
MetadataOnly(crate::domain::MetadataOnlyReason),
}
pub fn classify_stable_content(
path: &str,
targets: crate::domain::IndexTargets,
bytes: &[u8],
) -> StableContentAdmission {
classify_stable_content_with(path, targets, bytes, scan_secret_bytes)
}
pub fn classify_stable_content_with<F>(
path: &str,
_targets: crate::domain::IndexTargets,
bytes: &[u8],
scan: F,
) -> StableContentAdmission
where
F: FnOnce(&str, &[u8]) -> SecretScan,
{
if let Some(pointer) = detect_lfs_pointer(bytes) {
return StableContentAdmission::MetadataOnly(
crate::domain::MetadataOnlyReason::LfsPointer {
declared_oid: pointer.declared_oid,
declared_size: pointer.declared_size,
},
);
}
if decode_searchable_text(bytes).is_err() {
return StableContentAdmission::MetadataOnly(
crate::domain::MetadataOnlyReason::UnsupportedTextEncoding,
);
}
match scan(path, bytes) {
SecretScan::Clean => StableContentAdmission::Admitted,
SecretScan::Sensitive {
rule_ids,
finding_count,
..
} => StableContentAdmission::MetadataOnly(
crate::domain::MetadataOnlyReason::SensitiveContent {
rule_ids: rule_ids.into_iter().map(ToOwned::to_owned).collect(),
finding_count,
},
),
SecretScan::Indeterminate { .. } => StableContentAdmission::MetadataOnly(
crate::domain::MetadataOnlyReason::SensitiveContent {
rule_ids: vec![INDETERMINATE_RULE_ID.to_string()],
finding_count: 0,
},
),
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct GuardFailure {
pub policy_version: u32,
pub rule_ids: Vec<&'static str>,
pub finding_count: u32,
}
pub struct SafeHit<'a, T> {
inner: &'a T,
pub policy_version: u32,
}
impl<'a, T> SafeHit<'a, T> {
pub fn into_inner(self) -> &'a T {
self.inner
}
}
fn guard_visible_fields(fields: &[&str]) -> Result<(), GuardFailure> {
let mut rule_ids = Vec::new();
let mut finding_count = 0_u32;
for field in fields {
match scan_secret_bytes("external-field", field.as_bytes()) {
SecretScan::Clean => {}
SecretScan::Sensitive {
rule_ids: field_rule_ids,
finding_count: field_count,
..
} => {
finding_count = finding_count.saturating_add(field_count);
for rule_id in field_rule_ids {
if !rule_ids.contains(&rule_id) {
rule_ids.push(rule_id);
}
}
}
SecretScan::Indeterminate { .. } => {
if !rule_ids.contains(&INDETERMINATE_RULE_ID) {
rule_ids.push(INDETERMINATE_RULE_ID);
}
}
}
}
if rule_ids.is_empty() {
Ok(())
} else {
Err(GuardFailure {
policy_version: SECRET_POLICY_VERSION,
rule_ids,
finding_count,
})
}
}
pub fn guard_query(query: &str) -> Result<(), GuardFailure> {
guard_visible_fields(&[query])
}
pub fn guard_hit<'a, T>(hit: &'a T, fields: &[&str]) -> Result<SafeHit<'a, T>, GuardFailure> {
guard_visible_fields(fields)?;
Ok(SafeHit {
inner: hit,
policy_version: SECRET_POLICY_VERSION,
})
}
pub fn project_markdown_sections(
source: &crate::domain::SourceIdentity,
path: &str,
content_hash: &str,
symbols: &[crate::domain::SymbolRecord],
) -> Vec<crate::domain::KnowledgeUnit> {
let mut units: Vec<crate::domain::KnowledgeUnit> = Vec::new();
let mut parents: Vec<(u32, u32, String)> = Vec::new();
for symbol in symbols
.iter()
.filter(|symbol| symbol.kind == crate::domain::SymbolKind::Section)
{
while parents
.last()
.is_some_and(|(depth, _, _)| *depth >= symbol.depth)
{
parents.pop();
}
let parent = parents.last().map(|(_, index, _)| *index);
let segment = parents
.last()
.and_then(|(_, _, parent_name)| {
symbol
.name
.strip_prefix(parent_name)
.and_then(|suffix| suffix.strip_prefix('.'))
})
.unwrap_or(symbol.name.as_str())
.to_string();
let mut heading_path = parent
.and_then(|index| units.get(index as usize))
.map(|unit| unit.heading_path.clone())
.unwrap_or_default();
heading_path.push(segment);
let index = units.len() as u32;
units.push(crate::domain::KnowledgeUnit {
source: source.clone(),
path: path.to_string(),
content_hash: content_hash.to_string(),
kind: crate::domain::KnowledgeUnitKind::MarkdownSection,
heading_path,
byte_range: symbol.byte_range.0..symbol.byte_range.1,
line_range: symbol.line_range.0..symbol.line_range.1,
parent,
});
parents.push((symbol.depth, index, symbol.name.clone()));
}
units
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::{IndexTargets, MetadataOnlyReason};
fn runtime_canary() -> String {
["runtime", "-", "canary", "-", "segment"].concat()
}
#[test]
fn text_byte_matrix_handles_zero_lf_crlf_bom_multibyte_invalid_utf8_and_no_final_newline() {
for bytes in [
b"".as_slice(),
b"one\ntwo\n".as_slice(),
b"one\r\ntwo\r\n".as_slice(),
b"one\ntwo".as_slice(),
"one α\n二".as_bytes(),
] {
let decoded = decode_searchable_text(bytes).expect("valid UTF-8 must decode exactly");
assert_eq!(decoded.text.as_bytes(), bytes);
assert_eq!(decoded.leading_bytes, 0);
}
let mut bom = UTF8_BOM.to_vec();
bom.extend_from_slice("α\r\nlast".as_bytes());
let decoded = decode_searchable_text(&bom).expect("UTF-8 BOM must be accepted");
assert_eq!(decoded.text, "α\r\nlast");
assert_eq!(decoded.leading_bytes, 3);
assert!(decode_searchable_text(&[0xff, 0xfe, b'x']).is_err());
}
#[test]
fn detector_failure_fails_closed_and_discards_transient_bytes() {
let bytes = b"clean candidate".to_vec();
let decision = classify_stable_content_with(
"notes/guide.txt",
IndexTargets::Knowledge,
&bytes,
|_path, _bytes| SecretScan::Indeterminate {
reason: DetectorFailure::Internal,
},
);
drop(bytes);
assert!(matches!(
decision,
StableContentAdmission::MetadataOnly(MetadataOnlyReason::SensitiveContent {
finding_count: 0,
..
})
));
}
#[derive(Debug)]
struct CandidateHit {
path: String,
heading: String,
excerpt: String,
diagnostic: String,
source_label: String,
ranking: String,
}
#[test]
fn detector_positive_hit_is_withheld_whole_in_direct_and_ccr_paths() {
let canary = runtime_canary();
let hit = CandidateHit {
path: "notes/guide.md".to_string(),
heading: "Guide".to_string(),
excerpt: format!("password={canary}"),
diagnostic: "none".to_string(),
source_label: "current".to_string(),
ranking: "exact".to_string(),
};
let fields = [
hit.path.as_str(),
hit.heading.as_str(),
hit.excerpt.as_str(),
hit.diagnostic.as_str(),
hit.source_label.as_str(),
hit.ranking.as_str(),
];
let guarded = guard_hit(&hit, &fields);
let direct_visible = guarded.as_ref().ok().is_some();
let ccr_visible = guarded.ok().map(SafeHit::into_inner).is_some();
assert!(!direct_visible);
assert!(!ccr_visible);
}
#[test]
fn query_and_every_visible_hit_field_are_guarded_without_echo() {
let canary = runtime_canary();
let unsafe_field = format!("token={canary}");
assert!(guard_query(&unsafe_field).is_err());
for label in [
"path",
"heading",
"excerpt",
"diagnostic",
"source_label",
"ranking",
] {
let result = guard_hit(&label, &[unsafe_field.as_str()]);
assert!(result.is_err(), "guard missed visible field class {label}");
}
}
#[test]
fn lfs_pointer_parser_retains_only_declared_metadata() {
let pointer = b"version https://git-lfs.github.com/spec/v1\noid sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\nsize 42\n";
let metadata = detect_lfs_pointer(pointer).expect("valid pointer must be recognized");
assert!(metadata.declared_oid.is_some());
assert_eq!(metadata.declared_size, Some(42));
assert!(detect_lfs_pointer(b"ordinary prose").is_none());
}
#[test]
fn sensitive_path_policy_distinguishes_credentials_from_safe_templates() {
assert!(sensitive_path_rule(".env").is_some());
assert!(sensitive_path_rule("deploy/.env.production").is_some());
assert!(sensitive_path_rule(".ssh/id_ed25519").is_some());
assert!(sensitive_path_rule("state/prod.tfstate").is_some());
assert!(sensitive_path_rule(".env.example").is_none());
assert!(sensitive_path_rule("docs/secret-design.md").is_none());
}
#[test]
fn sensitive_path_rule_ignores_empty_and_dot_segments() {
for spelling in [
".aws//credentials",
".aws/./credentials",
"./.aws/credentials",
"infra/.kube/./config",
"infra\\.kube\\config",
".aws/credentials/",
] {
assert!(
sensitive_path_rule(spelling).is_some(),
"{spelling:?} must match the credential rule"
);
}
assert!(sensitive_path_rule("src/./lib.rs").is_none());
}
#[test]
fn sensitive_path_rule_folds_unicode_case() {
for spelling in [
".aws/credential\u{17F}",
".\u{212A}ube/config",
"keys/id_r\u{17F}a",
"deploy/.ENV",
".\u{212A}UBE/CONFIG",
] {
assert!(
sensitive_path_rule(spelling).is_some(),
"{spelling:?} must match its credential rule"
);
}
}
#[test]
fn sensitive_path_rule_at_judges_the_absolute_path_too() {
let root = std::path::Path::new("/home/user/.aws");
assert!(sensitive_path_rule("credentials").is_none());
assert_eq!(
sensitive_path_rule_at("credentials", &root.join("credentials")),
Some("path.cloud-credential-store")
);
let kube = std::path::Path::new(r"C:\Users\user\.kube");
assert!(sensitive_path_rule_at("config", &kube.join("config")).is_some());
assert!(
sensitive_path_rule_at(
"src/lib.rs",
std::path::Path::new("/home/user/repo/src/lib.rs")
)
.is_none()
);
}
fn mx_token() -> String {
["to", "ken"].concat()
}
fn mx_password() -> String {
["pass", "word"].concat()
}
fn mx_apikey() -> String {
["api", "key"].concat()
}
fn mx_deploy_token() -> String {
["DEPLOY_TO", "KEN"].concat()
}
fn mx_fallback_token() -> String {
["FALLBACK_TO", "KEN"].concat()
}
fn mx_value() -> String {
["a-long-", "literal-", "value"].concat()
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum MxVerdict {
Clean,
Sensitive(u32),
}
fn matrix_d1_rows() -> Vec<(&'static str, &'static str, String, MxVerdict)> {
let token = mx_token();
let password = mx_password();
let apikey = mx_apikey();
let deploy_token = mx_deploy_token();
let fallback_token = mx_fallback_token();
let v = mx_value();
vec![
(
"C5a js multi-declarator across lines",
"src/probe.js",
[
"const ",
&token,
" = \"placeholder\",\n backup = \"",
&v,
"\";",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"C5b js multi-declarator one line control",
"src/probe.js",
[
"const ",
&token,
" = \"placeholder\", backup = \"",
&v,
"\";",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"C5d rust multi-binding across lines",
"src/probe.rs",
[
"let ",
&token,
" = \"placeholder\",\n backup = \"",
&v,
"\";",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"G10b rust struct literal sibling field",
"src/probe.rs",
[
"let config = Config {\n ",
&apikey,
": resolve_from_environment(),\n banner_text: \"",
&v,
"\",\n};",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"D1-scan consumed walk passes over later match",
"src/probe.js",
[
"const ",
&token,
" = \"placeholder\",\n a1 = \"x1\",\n a2 = \"x2\",\n a3 = \"x3\",\n a4 = \"x4\";\nconst ",
&password,
" = \"",
&v,
"\";",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"P1 python tuple placeholder then literal",
"src/probe.py",
[&password, " = \"placeholder\", \"", &v, "\""].concat(),
MxVerdict::Sensitive(1),
),
(
"P4 python interpolation then literal",
"src/probe.py",
[
&token,
" = \"${{DEPLOY_HOST}}\", \"",
&v,
"\"",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"E2 python unquoted placeholder comma literal",
"src/probe.py",
[&password, " = placeholder, '", &v, "'"].concat(),
MxVerdict::Sensitive(1),
),
(
"K1 yaml apikey sibling two findings",
"config.yaml",
[
"auth: {",
&password,
": \"${CI_FALLBACK_TOKEN}\", ",
&apikey,
": \"",
&v,
"\"}",
]
.concat(),
MxVerdict::Sensitive(2),
),
(
"K2 yaml bearer sibling walk only",
"config.yaml",
[
"auth: {",
&password,
": \"${CI_FALLBACK_TOKEN}\", bearer: \"",
&v,
"\"}",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"K3 yaml credential sibling walk only",
"config.yaml",
[
"auth: {",
&password,
": \"${CI_FALLBACK_TOKEN}\", credential: \"",
&v,
"\"}",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"K4 yaml accesskey sibling walk only",
"config.yaml",
[
"auth: {",
&password,
": \"${CI_FALLBACK_TOKEN}\", accesskey: \"",
&v,
"\"}",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"F5 env comma-joined list one key",
"deploy.env",
[
&fallback_token,
"=\"${CI_TOKEN}\", \"",
&v,
"\"",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"F3 env one-comma evasion variant",
"deploy.env",
[
&deploy_token,
"=${CI_FALLBACK_TOKEN}, \"",
&v,
"\"",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"F3ctl env no-comma control",
"deploy.env",
[
&deploy_token,
"=${CI_FALLBACK_TOKEN} \"",
&v,
"\"",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"Y1u yaml flow unquoted capture swallows comma",
"config.yaml",
["{", &token, ": ${TOKEN_NAME}, banner: \"", &v, "\"}"].concat(),
MxVerdict::Sensitive(1),
),
(
"Y1q yaml flow quoted sibling",
"config.yaml",
[
"{",
&token,
": \"${TOKEN_NAME}\", banner: \"",
&v,
"\"}",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"A3 yaml note sibling walk coverage",
"config.yaml",
[
"auth: {",
&token,
": \"${TOKEN_NAME}\", note: \"",
&v,
"\"}",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"B1 toml bracketed array",
"config.toml",
[&apikey, " = [\"placeholder\", \"", &v, "\"]"].concat(),
MxVerdict::Sensitive(1),
),
(
"X3 js unquoted capture before fence",
"src/probe.js",
["const ", &token, " = placeholder\"", &v, "\", \"x"].concat(),
MxVerdict::Clean,
),
(
"X1 js apostrophe in placeholder literal",
"src/probe.js",
[
"const ",
&token,
" = \"your-org's-token\", \"",
&v,
"\", \"x",
]
.concat(),
MxVerdict::Clean,
),
]
}
#[test]
fn comma_continuation_d1_matrix_pins() {
let mut failures = Vec::new();
for (id, path, body, expected) in matrix_d1_rows() {
let actual = match scan_secret_bytes(path, body.as_bytes()) {
SecretScan::Clean => MxVerdict::Clean,
SecretScan::Sensitive { finding_count, .. } => MxVerdict::Sensitive(finding_count),
SecretScan::Indeterminate { reason } => {
failures.push(format!("{id}: unexpected Indeterminate: {reason:?}"));
continue;
}
};
if actual != expected {
failures.push(format!("{id}: expected {expected:?}, got {actual:?}"));
}
}
assert!(
failures.is_empty(),
"matrix rows off expectation: {failures:?}"
);
}
fn bracketed_array_rows() -> Vec<(&'static str, &'static str, String, MxVerdict)> {
let token = mx_token();
let apikey = mx_apikey();
let deploy_token = mx_deploy_token();
let v = mx_value();
vec![
(
"AR1 toml single-element array",
"config.toml",
[&apikey, " = [\"", &v, "\"]"].concat(),
MxVerdict::Sensitive(1),
),
(
"AR2 toml short-first element",
"config.toml",
[&apikey, " = [\"dev\", \"", &v, "\"]"].concat(),
MxVerdict::Sensitive(1),
),
(
"AR3 toml all-short elements",
"config.toml",
[&apikey, " = [\"dev\", \"shrt\"]"].concat(),
MxVerdict::Clean,
),
(
"AR4 yaml flow-map key not captured",
"probe.yaml",
["{", &token, ": \"ab\", environment: production}"].concat(),
MxVerdict::Clean,
),
(
"AR5 toml placeholder-only element",
"config.toml",
[&apikey, " = [\"${", &deploy_token, "}\"]"].concat(),
MxVerdict::Clean,
),
(
"AR6 toml placeholder then payload",
"config.toml",
[&apikey, " = [\"${", &deploy_token, "}\", \"", &v, "\"]"].concat(),
MxVerdict::Sensitive(1),
),
(
"AR7 js array on code path",
"src/probe.js",
["const ", &token, " = [\"dev\", \"", &v, "\"];"].concat(),
MxVerdict::Sensitive(1),
),
(
"AR8 yaml flow sequence",
"probe.yaml",
[&token, ": [\"dev\", \"", &v, "\"]"].concat(),
MxVerdict::Sensitive(1),
),
]
}
#[test]
fn bracketed_array_capture_pins() {
let mut failures = Vec::new();
for (id, path, body, expected) in bracketed_array_rows() {
let actual = match scan_secret_bytes(path, body.as_bytes()) {
SecretScan::Clean => MxVerdict::Clean,
SecretScan::Sensitive { finding_count, .. } => MxVerdict::Sensitive(finding_count),
SecretScan::Indeterminate { reason } => {
failures.push(format!("{id}: unexpected Indeterminate: {reason:?}"));
continue;
}
};
if actual != expected {
failures.push(format!("{id}: expected {expected:?}, got {actual:?}"));
}
}
assert!(
failures.is_empty(),
"array rows off expectation: {failures:?}"
);
}
fn mx_real() -> String {
["Zq8r", "Lm3v", "Tx7w", "Pk2n", "Hy5s"].concat()
}
fn scan_verdict(path: &str, body: &str) -> Result<MxVerdict, String> {
match scan_secret_bytes(path, body.as_bytes()) {
SecretScan::Clean => Ok(MxVerdict::Clean),
SecretScan::Sensitive { finding_count, .. } => Ok(MxVerdict::Sensitive(finding_count)),
SecretScan::Indeterminate { reason } => Err(format!("Indeterminate: {reason:?}")),
}
}
fn precision_rows() -> Vec<(&'static str, &'static str, String, MxVerdict)> {
let token = mx_token();
let token_camel = ["To", "ken"].concat();
let apikey_camel = ["Api", "Key"].concat();
let secret_kw = ["sec", "ret"].concat();
let real40 = [mx_real(), mx_real()].concat();
let password = mx_password();
let apikey = mx_apikey();
let v = mx_value();
let real = mx_real();
vec![
(
"PA1 rust split closes keyword literal",
"src/probe.rs",
[
"let first_", &token, " = first.split(\"", &token,
"=\").nth(1).expect(\"", &token, " in first URL\");\n",
]
.concat(),
MxVerdict::Clean,
),
(
"PA2 control: literal after the closing quote",
"src/probe.rs",
[
"let url = base.split(\"", &token, "=\").nth(1).unwrap_or(\"", &real,
"\");\n",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"PA3 control: char-literal quote then real assignment",
"src/probe.rs",
[
"let q = '\"'; let ", &token, " = \"", &real, "\"; // \"\n",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"PB1 rust struct field then doc comment",
"src/probe.rs",
[
"pub struct Config {\n pub gemini_", &apikey,
": Option<String>,\n /// Gemini model ID (e.g. \"gemini-2.5-flash\").\n pub gemini_model: Option<String>,\n}\n",
]
.concat(),
MxVerdict::Clean,
),
(
"PB2 rust struct field then attribute",
"src/probe.rs",
[
"pub struct Config {\n pub openai_", &apikey,
": Option<String>,\n #[serde(rename = \"", &v,
"\")]\n pub model: Option<String>,\n}\n",
]
.concat(),
MxVerdict::Clean,
),
(
"PB4 js declarator after a comment line (accepted)",
"src/probe.js",
[
"const ", &token, " = compute(),\n // fallback\n backup = \"", &v,
"\";\n",
]
.concat(),
MxVerdict::Clean,
),
(
"PC1 env example fill-in placeholder",
"server/.env.example",
[&apikey.to_uppercase(), "=<<FILL_IN: gemini key>>\nPORT=8080\n"].concat(),
MxVerdict::Clean,
),
(
"PC2 toml quoted fill-in placeholder",
"config.toml",
[&apikey, " = \"<<FILL_IN: gemini key>>\"\n"].concat(),
MxVerdict::Clean,
),
(
"PC3 control: fill-in then literal element",
"config.toml",
[&apikey, " = [\"<<FILL_IN: gemini key>>\", \"", &v, "\"]\n"].concat(),
MxVerdict::Sensitive(1),
),
(
"PD1 rust lifetime before keyword parameter",
"src/probe.rs",
[
"pub fn spawn_worker_on<F>(&self, name: &'static str, ", &token,
": CancellationToken, work: F) -> bool {\n",
]
.concat(),
MxVerdict::Clean,
),
(
"PD2 control: python apostrophe still opens a string",
"src/probe.py",
["url = 'https://host/?", &token, "=", &v, "'\n"].concat(),
MxVerdict::Sensitive(1),
),
(
"PE1 uri pass placeholder in comment",
"src/probe.rs",
["// userinfo in scheme://user:", "pass", "@host/db\n"].concat(),
MxVerdict::Clean,
),
(
"PE2 uri redacted placeholder",
"src/probe.rs",
["// postgres://u:", "<redacted>", "@host/db\n"].concat(),
MxVerdict::Clean,
),
(
"PE3 uri password xxxx secret placeholders",
"notes.txt",
[
"a://admin:",
&password,
"@localhost\nb://admin:",
"xxxx",
"@example.com\nc://admin:",
"secret",
"@${DB_HOST}\n",
]
.concat(),
MxVerdict::Clean,
),
(
"PF1 rust doc comment code span (known ceiling)",
"src/probe.rs",
[
"/// So `--", &password, "=postgres://u:pw@host` is fully opaque.\n",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"PG1 rust doc link to a token path",
"src/probe.rs",
[
" /// and must poll [`Cancellation", &token.to_uppercase()[..1], &token[1..],
"::is_cancelled`] wherever it would\n",
]
.concat(),
MxVerdict::Clean,
),
(
"PG2 control: path call carrying a literal",
"src/probe.rs",
["let t = ", &token, "::from_secret_value(\"", &real, "\");\n"].concat(),
MxVerdict::Sensitive(1),
),
(
"PG3 control: path call whose literal is on the next line",
"src/probe.rs",
[
"let t = ", &token, "::from_secret_value(\n \"", &real, "\",\n);\n",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"F1a cpp constant with its literal on the next line",
"src/probe.cpp",
[
"const char* ",
&apikey_camel,
"::kDefault =\n \"",
&real40,
"\";\n",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"F1b rust builder chain on a continuation line",
"src/probe.rs",
[
"let client = ",
&token_camel,
"::builder()\n .secret(\"",
&real40,
"\")\n .build();\n",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"F1c ruby constant with its literal on the next line",
"lib/probe.rb",
[&token_camel, "::DEFAULT_VALUE =\n \"", &real40, "\"\n"].concat(),
MxVerdict::Sensitive(1),
),
(
"F6a angle group followed by a value",
"src/probe.rs",
[
"let u = \"https://h/?",
&token,
"=<v1>",
&real40,
"\";\n",
]
.concat(),
MxVerdict::Sensitive(1),
),
(
"F6b heredoc opener with the value on the next line",
"infra/main.tf",
[&secret_kw, " = <<-EOT_LONGNAME\n", &real40, "\nEOT_LONGNAME\n"].concat(),
MxVerdict::Sensitive(1),
),
(
"F6c env angle group around a real value",
"deploy.env",
[&apikey.to_uppercase(), "=<", &real40, ">\n"].concat(),
MxVerdict::Sensitive(1),
),
(
"F6d yaml quoted angle group around a real value",
"config.yaml",
[&password, ": \"<", &real40, ">\"\n"].concat(),
MxVerdict::Sensitive(1),
),
(
"F6e env angle placeholder word",
"deploy.env",
[&apikey.to_uppercase(), "=<your-api-", "key>\n"].concat(),
MxVerdict::Clean,
),
(
"F7a changeme on a production host",
"config.yaml",
["url: postgres://admin:", "changeme", "@prod-db.internal:5432/app\n"].concat(),
MxVerdict::Sensitive(1),
),
(
"F7b password on a production host",
"config.yaml",
["broker: amqp://app:", &password, "@mq-prod\n"].concat(),
MxVerdict::Sensitive(1),
),
(
"AWS1 python secret access key from environment",
"src/probe.py",
["aws_", &secret_kw, "_access_key = os.environ.get(KEY_NAME)\n"].concat(),
MxVerdict::Clean,
),
(
"AWS2 python secret access key literal",
"src/probe.py",
["aws_", &secret_kw, "_access_key = \"", &real40, "\"\n"].concat(),
MxVerdict::Sensitive(1),
),
(
"PJ1 json quoted key (known false negative)",
"config.json",
["{\"", &password, "\": \"hunter2hunter2\"}\n"].concat(),
MxVerdict::Clean,
),
]
}
#[test]
fn precision_ruling_matrix_pins() {
let mut failures = Vec::new();
for (id, path, body, expected) in precision_rows() {
match scan_verdict(path, &body) {
Ok(actual) if actual == expected => {}
Ok(actual) => failures.push(format!("{id}: expected {expected:?}, got {actual:?}")),
Err(error) => failures.push(format!("{id}: {error}")),
}
}
assert!(
failures.is_empty(),
"precision rows off expectation: {failures:?}"
);
}
#[test]
fn true_positive_corpus_stays_caught() {
let token = mx_token();
let apikey = mx_apikey();
let real = mx_real();
let sk = ["sk-", &real, &real].concat();
let ghp = ["gh", "p_", &real, &real[..16]].concat();
let pem = [
"-----BEGIN RSA ",
"PRIVATE KEY-----\nMIIEowIBAAKCAQEA",
&real,
"\n-----END RSA ",
"PRIVATE KEY-----\n",
]
.concat();
let bearer = ["Author", "ization: Bearer ", &real, &real[..12]].concat();
let aws_secret = [&real, "/", &real[..9], "+", &real[..9]].concat();
let rows: Vec<(&str, &str, String, &str)> = vec![
(
"github token rust",
"src/probe.rs",
["let t = \"", &ghp, "\";\n"].concat(),
"secret.provider-token",
),
(
"github token yaml",
"ci.yaml",
["gh: ", &ghp, "\n"].concat(),
"secret.provider-token",
),
(
"pem private key",
"certs/server.txt",
pem,
"secret.private-key-envelope",
),
(
"authorization bearer",
"requests.http",
["GET /\n", &bearer, "\n"].concat(),
"secret.authorization-header",
),
(
"sk key rust",
"src/probe.rs",
["let ", &apikey, " = \"", &sk, "\";\n"].concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"sk key python",
"src/probe.py",
[&apikey, " = \"", &sk, "\"\n"].concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"sk key toml",
"config.toml",
[&apikey, " = \"", &sk, "\"\n"].concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"sk key yaml",
"config.yaml",
[&apikey, ": \"", &sk, "\"\n"].concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"sk key env",
"deploy.env",
[&apikey.to_uppercase(), "=", &sk, "\n"].concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"uri real password",
"src/probe.rs",
["let url = \"postgres://user:", &real, "@db/prod\";\n"].concat(),
URI_CREDENTIALS_RULE_ID,
),
(
"fp-shape split then literal",
"src/probe.rs",
[
"let url = base.split(\"",
&token,
"=\").nth(1).unwrap_or(\"",
&real,
"\");\n",
]
.concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"fp-shape struct literal field then doc comment",
"src/probe.rs",
[
"let c = Config {\n ",
&apikey,
": \"",
&real,
"\".into(),\n /// doc\n other: 1,\n};\n",
]
.concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"fp-shape test struct literal",
"tests/api.rs",
[" admin_", &token, ": \"", &real, "\".into(),\n"].concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"fp-shape env example real value",
"server/.env.example",
[&apikey.to_uppercase(), "=", &sk, "\n"].concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"fp-shape lifetime then literal",
"src/probe.rs",
[
"fn f(name: &'static str) { let ",
&token,
" = \"",
&real,
"\"; }\n",
]
.concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"fp-shape lifetime then keyword in literal",
"src/probe.rs",
[
"const URL: &'static str = \"https://h/?",
&token,
"=",
&real,
"\";\n",
]
.concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"fp-shape uri placeholder slot real value",
"src/probe.rs",
["// scheme://user:", &real, "@authority\n"].concat(),
URI_CREDENTIALS_RULE_ID,
),
(
"fp-shape fill-in slot real value",
"config.toml",
[&apikey, " = \"", &sk, "\"\n"].concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"aws secret access key ini",
"deploy/aws.ini",
[
"[default]\naws_",
"sec",
"ret_access_key = ",
&aws_secret,
"\n",
]
.concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
(
"aws secret access key ini upper",
"deploy/profile.cfg",
["AWS_", "SEC", "RET_ACCESS_KEY=", &aws_secret, "\n"].concat(),
CONTEXT_ASSIGNMENT_RULE_ID,
),
];
let mut failures = Vec::new();
for (id, path, body, rule) in rows {
match scan_secret_bytes(path, body.as_bytes()) {
SecretScan::Sensitive {
rule_ids,
line_ranges,
..
} if rule_ids.contains(&rule) => {
if line_ranges.is_empty() {
failures.push(format!("{id}: caught but no finding line recorded"));
}
}
other => failures.push(format!(
"{id}: expected {rule}, got {:?}",
scan_kind(&other)
)),
}
}
assert!(failures.is_empty(), "true positives missed: {failures:?}");
}
fn scan_kind(scan: &SecretScan) -> String {
match scan {
SecretScan::Clean => "Clean".to_string(),
SecretScan::Sensitive { rule_ids, .. } => format!("Sensitive{rule_ids:?}"),
SecretScan::Indeterminate { reason } => format!("Indeterminate({reason:?})"),
}
}
#[test]
fn findings_carry_one_based_lines() {
let body = ["fn a() {}\nlet ", &mx_token(), " = \"", &mx_real(), "\";\n"].concat();
let SecretScan::Sensitive { line_ranges, .. } =
scan_secret_bytes("src/probe.rs", body.as_bytes())
else {
panic!("fixture must be caught");
};
assert_eq!(line_ranges, vec![(2, 2)]);
}
#[test]
fn finding_ranges_merge_across_rules() {
let uri = ["scheme://u:", &mx_real(), "@db.prod\n"].concat();
let short = [&["p", "wd"].concat(), "=", "a1b2c3d4", "\n"].concat();
let body = format!("{uri}{short}").repeat(3);
let SecretScan::Sensitive {
finding_count,
line_ranges,
..
} = scan_secret_bytes("deploy.env", body.as_bytes())
else {
panic!("fixture must be caught");
};
assert_eq!(finding_count, 6);
assert_eq!(line_ranges, vec![(1, 6)]);
}
#[test]
fn finding_ranges_stop_after_the_kept_count() {
let short = [&["p", "wd"].concat(), "=", "a1b2c3d4", "\n\n"].concat();
let SecretScan::Sensitive {
finding_count,
line_ranges,
..
} = scan_secret_bytes("deploy.env", short.repeat(15).as_bytes())
else {
panic!("fixture must be caught");
};
assert_eq!(finding_count, 15);
let expected = (0..FINDING_LINE_RANGES_KEPT as u32)
.map(|index| (2 * index + 1, 2 * index + 1))
.collect::<Vec<_>>();
assert_eq!(line_ranges, expected);
}
#[test]
fn dense_findings_at_the_scan_budget_scan_in_linear_time() {
let finding = [&["p", "wd"].concat(), "=", "a1b2c3d4", "\n"].concat();
let count = (SECRET_SCAN_MAX_BYTES - 64) / finding.len();
let body = finding.repeat(count);
let started = std::time::Instant::now();
let SecretScan::Sensitive {
finding_count,
line_ranges,
..
} = scan_secret_bytes("deploy.env", body.as_bytes())
else {
panic!("fixture must be caught");
};
let elapsed = started.elapsed();
assert_eq!(finding_count as usize, count);
assert_eq!(line_ranges, vec![(1, u32::try_from(count).unwrap())]);
assert!(
elapsed < std::time::Duration::from_secs(20),
"{count} findings took {elapsed:?}"
);
}
#[test]
fn markdown_sections_project_to_knowledge_units_without_a_duplicate_store() {
let source = crate::domain::SourceIdentity {
repository_id: crate::domain::RepositoryId::new("repository-fixture"),
source_id: crate::domain::SourceId::new("source-fixture"),
location: crate::domain::SourceLocation::WorkingTree {
worktree_id: "worktree-fixture".to_string(),
},
};
let result = crate::parsing::process_file_with_classification(
"README.md",
b"# Root\nintro\n## Child.with.dot\nbody\n",
crate::domain::LanguageId::Markdown,
crate::domain::FileClassification::for_indexed_path(
"README.md",
IndexTargets::Knowledge,
),
);
let units = project_markdown_sections(
&source,
&result.relative_path,
&result.content_hash,
&result.symbols,
);
assert_eq!(units.len(), result.symbols.len());
assert_eq!(units.len(), 2);
assert_eq!(units[0].heading_path, ["Root"]);
assert_eq!(units[0].parent, None);
assert_eq!(units[1].heading_path, ["Root", "Child.with.dot"]);
assert_eq!(units[1].parent, Some(0));
assert!(units.iter().all(|unit| {
unit.kind == crate::domain::KnowledgeUnitKind::MarkdownSection
&& unit.path == result.relative_path
&& unit.content_hash == result.content_hash
}));
}
}