use std::collections::{BTreeSet, HashMap, HashSet};
use std::path::Path;
use std::sync::LazyLock;
use regex::Regex;
use crate::analysis::{line_column, push};
use crate::scanner::SourceFile;
use crate::{Diagnostic, Severity};
static TOKEN_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"[A-Za-z0-9_]+|[^\sA-Za-z0-9_]").expect("valid token regex"));
static LINK_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\[[^\]]+\]\(([^)#?]+)(?:#[^)]+)?\)").expect("valid link regex"));
static CODE_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"`([^`\n]+)`").expect("valid inline code regex"));
static PATH_RE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r#"(?x)(?:^|[\s(\[\{\"'])( (?:\.?\.?/)?(?:src|docs|scripts|tests?|config|packages|apps|crates|lib|bin|database|migrations|skills)/[A-Za-z0-9_./@+~-]+ | [A-Za-z0-9_.@+-]+/[A-Za-z0-9_./@+~-]+\.(?:md|mdc|rs|ts|tsx|js|jsx|py|go|toml|json|ya?ml|sh|fish|prisma|sql) )"#).expect("valid path regex")
});
static SECRET_ASSIGNMENT_RE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r#"(?i)\b(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|GITHUB_TOKEN|GITLAB_TOKEN|AWS_SECRET_ACCESS_KEY|API_KEY|ACCESS_TOKEN|CLIENT_SECRET)\b\s*[:=]\s*[\"']?([^\s\"',}\]]+)"#).expect("valid secret regex")
});
static BEARER_RE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"(?i)\b(?:authorization\s*:\s*)?bearer\s+([A-Za-z0-9._~+/=-]{16,})")
.expect("valid bearer regex")
});
static PREFIX_SECRET_RE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"\b((?:sk-(?:proj-)?|gh[pousr]_|github_pat_)[A-Za-z0-9_-]{12,})\b")
.expect("valid credential prefix regex")
});
#[derive(Debug, Clone)]
struct Sentence<'a> {
file: &'a SourceFile,
line: usize,
text: String,
normalized: Vec<String>,
}
pub fn approximate_tokens(text: &str) -> usize {
TOKEN_RE.find_iter(text).count()
}
pub fn check_file(root: &Path, file: &SourceFile, diagnostics: &mut Vec<Diagnostic>) {
check_references(root, file, diagnostics);
check_secrets(file, diagnostics);
check_context(file, diagnostics);
}
fn check_context(file: &SourceFile, diagnostics: &mut Vec<Diagnostic>) {
let tokens = approximate_tokens(&file.content);
if tokens >= 6_000 {
push(
diagnostics,
Diagnostic::new(
"AGL301",
Severity::Warning,
&file.relative_path,
format!("This file is approximately {tokens} tokens and may consume significant agent context."),
)
.suggest("Move detailed reference material to on-demand documentation and keep durable instructions concise."),
);
}
let body = strip_frontmatter(&file.content);
let mut offset = file.content.len().saturating_sub(body.len());
for block in body.split("\n\n") {
if block.chars().count() >= 1_200 && !block.trim_start().starts_with("```") {
let (line, column) = line_column(&file.content, offset);
push(
diagnostics,
Diagnostic::new(
"AGL303",
Severity::Info,
&file.relative_path,
format!(
"This instruction block is {} characters long.",
block.chars().count()
),
)
.at(line, column)
.suggest("Split this block into short, actionable instructions."),
);
}
offset += block.len() + 2;
}
}
fn check_references(root: &Path, file: &SourceFile, diagnostics: &mut Vec<Diagnostic>) {
let mut references = BTreeSet::new();
for capture in LINK_RE.captures_iter(&file.content) {
if let Some(value) = capture.get(1) {
references.insert((value.start(), clean_path(value.as_str())));
}
}
for capture in CODE_RE.captures_iter(&file.content) {
let Some(value) = capture.get(1) else {
continue;
};
if plausible_path(value.as_str()) {
references.insert((value.start(), clean_path(value.as_str())));
}
}
for capture in PATH_RE.captures_iter(&file.content) {
if let Some(value) = capture.get(1) {
references.insert((value.start(), clean_path(value.as_str())));
}
}
for (offset, reference) in references {
if reference.is_empty()
|| reference.contains('*')
|| reference.starts_with("http://")
|| reference.starts_with("https://")
|| reference.starts_with('#')
|| reference.contains("${")
|| reference == "/"
{
continue;
}
let candidate = Path::new(&reference);
let resolved = if candidate.is_absolute() {
candidate.to_path_buf()
} else {
root.join(candidate)
};
if resolved.exists() {
continue;
}
let (line, column) = line_column(&file.content, offset);
let rule = if matches!(
candidate.extension().and_then(|value| value.to_str()),
Some("sh" | "fish" | "py")
) && reference.starts_with("scripts/")
{
"AGL203"
} else {
"AGL202"
};
let severity = if rule == "AGL203" {
Severity::Error
} else {
Severity::Warning
};
let display = reference.trim_end_matches(['.', ',', ':', ';']);
let mut diagnostic = Diagnostic::new(
rule,
severity,
&file.relative_path,
format!("Referenced path `{display}` does not exist in this repository."),
)
.at(line, column)
.reference(display);
if let Some(close) = unambiguous_path_match(root, display) {
diagnostic = diagnostic.suggest(format!("Did you mean `{close}`?"));
}
push(diagnostics, diagnostic);
}
}
fn plausible_path(value: &str) -> bool {
let value = value.trim();
!value.contains(' ')
&& !value.starts_with('-')
&& !value.contains("://")
&& (value.starts_with("./")
|| value.starts_with("../")
|| value.contains('/') && Path::new(value).extension().is_some()
|| [
"src/",
"docs/",
"scripts/",
"tests/",
"database/",
"skills/",
]
.iter()
.any(|prefix| value.starts_with(prefix)))
}
fn clean_path(value: &str) -> String {
value
.trim()
.trim_matches(['`', '"', '\'', '(', ')', '[', ']'])
.trim_end_matches(['.', ',', ':', ';'])
.split('#')
.next()
.unwrap_or_default()
.to_owned()
}
fn unambiguous_path_match(root: &Path, missing: &str) -> Option<String> {
let missing_path = Path::new(missing);
let parent = missing_path.parent()?;
let name = missing_path.file_name()?.to_str()?;
let directory = root.join(parent);
let entries = std::fs::read_dir(directory).ok()?;
let mut matches = entries
.filter_map(Result::ok)
.filter_map(|entry| entry.file_name().into_string().ok())
.filter(|candidate| strsim::levenshtein(candidate, name) <= 2);
let first = matches.next()?;
matches
.next()
.is_none()
.then(|| parent.join(first).to_string_lossy().into_owned())
}
fn check_secrets(file: &SourceFile, diagnostics: &mut Vec<Diagnostic>) {
let mut matches = Vec::new();
for regex in [&*SECRET_ASSIGNMENT_RE, &*BEARER_RE, &*PREFIX_SECRET_RE] {
for capture in regex.captures_iter(&file.content) {
if let Some(value) = capture.get(1).filter(|value| likely_secret(value.as_str())) {
matches.push((value.start(), redact(value.as_str())));
}
}
}
matches.sort();
matches.dedup();
for (offset, masked) in matches {
let (line, column) = line_column(&file.content, offset);
push(
diagnostics,
Diagnostic::new(
"AGL401",
Severity::Error,
&file.relative_path,
format!("Potential secret committed inside agent configuration (`{masked}`)."),
)
.at(line, column)
.suggest("Remove the value, rotate the credential, and reference an environment variable instead."),
);
}
}
fn likely_secret(value: &str) -> bool {
let upper = value.to_ascii_uppercase();
if value.len() < 12
|| value.contains("${")
|| value.contains("process.env")
|| value.starts_with('<')
|| upper.contains("YOUR_")
|| upper.contains("EXAMPLE")
|| upper.contains("PLACEHOLDER")
|| upper.contains("REDACTED")
|| upper == "API_KEY"
|| value
.chars()
.all(|character| character == 'x' || character == '*')
{
return false;
}
value
.chars()
.any(|character| character.is_ascii_alphabetic())
&& value.chars().any(|character| character.is_ascii_digit())
}
fn redact(value: &str) -> String {
let characters: Vec<char> = value.chars().collect();
if characters.len() <= 8 {
return "[redacted]".into();
}
let start: String = characters.iter().take(6).collect();
let end: String = characters.iter().rev().take(3).rev().collect();
format!("{start}...{end}")
}
pub fn check_cross_file(files: &[SourceFile], diagnostics: &mut Vec<Diagnostic>) -> usize {
let sentences: Vec<Sentence<'_>> = files.iter().flat_map(sentences).collect();
let mut duplicated_tokens = 0;
for left_index in 0..sentences.len() {
for right in &sentences[left_index + 1..] {
let left = &sentences[left_index];
if left.file.relative_path == right.file.relative_path
|| left.normalized.len() < 5
|| right.normalized.len() < 5
{
continue;
}
let score = similarity(&left.normalized, &right.normalized);
if score >= 0.78 {
duplicated_tokens += left.normalized.len().min(right.normalized.len());
push(
diagnostics,
Diagnostic::new(
"AGL302",
Severity::Warning,
&right.file.relative_path,
format!(
"Instruction duplicates {}:{}.",
left.file.relative_path.display(),
left.line
),
)
.at(right.line, 1)
.reference(format!(
"{}:{}",
left.file.relative_path.display(),
left.line
))
.suggest("Keep shared guidance in one canonical instruction file."),
);
}
}
}
check_contradictions(&sentences, diagnostics);
duplicated_tokens
}
fn sentences(file: &SourceFile) -> Vec<Sentence<'_>> {
let mut result = Vec::new();
let body = strip_frontmatter(&file.content);
let mut offset = file.content.len().saturating_sub(body.len());
for line in body.lines() {
for part in line.split(['.', '!', '?', ';']) {
let text = part.trim().trim_start_matches(|character: char| {
matches!(character, '-' | '*' | '#' | '0'..='9' | '.' | ' ')
});
let normalized = normalize_words(text);
if normalized.len() >= 3 && looks_instructional(text) {
result.push(Sentence {
file,
line: line_column(&file.content, offset).0,
text: text.to_owned(),
normalized,
});
}
}
offset += line.len() + 1;
}
result
}
fn looks_instructional(text: &str) -> bool {
let lower = text.to_ascii_lowercase();
[
"always ",
"never ",
"must ",
"do not ",
"don't ",
"use ",
"avoid ",
"prefer ",
"only ",
"required",
"forbidden",
"run ",
"before ",
"may freely ",
]
.iter()
.any(|marker| lower.contains(marker))
}
fn normalize_words(text: &str) -> Vec<String> {
static WORDS: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"[a-z0-9_./-]+").expect("valid words regex"));
static STOP: &[&str] = &[
"a", "an", "the", "to", "please", "you", "we", "should", "changes", "change",
];
WORDS
.find_iter(&text.to_ascii_lowercase())
.map(|item| item.as_str().to_owned())
.filter(|word| !STOP.contains(&word.as_str()))
.collect()
}
fn similarity(left: &[String], right: &[String]) -> f64 {
let left_set: HashSet<&str> = left.iter().map(String::as_str).collect();
let right_set: HashSet<&str> = right.iter().map(String::as_str).collect();
let intersection =
f64::from(u32::try_from(left_set.intersection(&right_set).count()).unwrap_or(u32::MAX));
let union = f64::from(u32::try_from(left_set.union(&right_set).count()).unwrap_or(u32::MAX));
let jaccard = if union == 0.0 {
0.0
} else {
intersection / union
};
let left_joined = left.join(" ");
let right_joined = right.join(" ");
let edit = strsim::normalized_levenshtein(&left_joined, &right_joined);
jaccard.max(edit)
}
#[derive(Debug)]
struct Directive {
negative: bool,
subject: String,
}
fn check_contradictions(sentences: &[Sentence<'_>], diagnostics: &mut Vec<Diagnostic>) {
let mut directives: HashMap<String, Vec<(&Sentence<'_>, bool)>> = HashMap::new();
for sentence in sentences {
if let Some(directive) = directive(&sentence.text) {
directives
.entry(directive.subject)
.or_default()
.push((sentence, directive.negative));
}
}
for entries in directives.values() {
for (index, (left, left_negative)) in entries.iter().enumerate() {
for (right, right_negative) in &entries[index + 1..] {
if left_negative == right_negative {
continue;
}
push(
diagnostics,
Diagnostic::new(
"AGL602",
Severity::Warning,
&right.file.relative_path,
format!(
"Instruction contradicts {}:{} (`{}`).",
left.file.relative_path.display(),
left.line,
concise(&left.text)
),
)
.at(right.line, 1)
.reference(format!(
"{}:{}",
left.file.relative_path.display(),
left.line
))
.suggest("Choose one policy and remove or scope the conflicting instruction."),
);
}
}
}
}
fn directive(text: &str) -> Option<Directive> {
let lower = text.to_ascii_lowercase();
let negative_markers = [
"never ",
"must not ",
"do not ",
"don't ",
"avoid ",
"forbidden",
"skip ",
];
let positive_markers = [
"always ",
"must ",
"required",
"use ",
"do ",
"only ",
"may freely ",
"run ",
];
let negative = negative_markers.iter().any(|marker| lower.contains(marker));
if !negative && !positive_markers.iter().any(|marker| lower.contains(marker)) {
return None;
}
let mut words = normalize_words(&lower);
for word in &mut words {
if word == "modify" || word == "editing" || word == "modifying" {
*word = "edit".into();
}
if word == "tests" {
*word = "test".into();
}
}
words.retain(|word| {
![
"always",
"never",
"must",
"not",
"do",
"don't",
"avoid",
"forbidden",
"required",
"use",
"only",
"may",
"freely",
"skip",
"run",
"instead",
"before",
"committing",
"files",
"under",
]
.contains(&word.as_str())
});
if words.is_empty() {
return None;
}
words.sort();
words.dedup();
Some(Directive {
negative,
subject: words.join(" "),
})
}
fn concise(text: &str) -> String {
const LIMIT: usize = 80;
if text.chars().count() <= LIMIT {
text.to_owned()
} else {
format!("{}…", text.chars().take(LIMIT).collect::<String>())
}
}
fn strip_frontmatter(text: &str) -> &str {
if let Some(stripped) = text.strip_prefix("---\n") {
if let Some(end) = stripped.find("\n---") {
let after = 4 + end + 4;
return text.get(after..).unwrap_or_default();
}
} else if let Some(stripped) = text.strip_prefix("---\r\n") {
if let Some(end) = stripped.find("\r\n---") {
let after = 5 + end + 6;
return text.get(after..).unwrap_or_default();
}
}
text
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn secret_redaction_never_reveals_full_value() {
let value = "sk-proj-abc123456789xyz";
let masked = redact(value);
assert!(!masked.contains(value));
assert_eq!(masked, "sk-pro...xyz");
}
#[test]
fn placeholders_are_not_secrets() {
assert!(!likely_secret("${OPENAI_API_KEY}"));
assert!(!likely_secret("YOUR_API_KEY"));
assert!(!likely_secret("example-token-12345"));
}
#[test]
fn reordered_instruction_is_similar() {
let left = normalize_words("Always run npm test before committing changes");
let right = normalize_words("Before committing, always run npm test");
assert!(similarity(&left, &right) >= 0.78);
}
}