use std::{
collections::BTreeMap,
io::{self, Read, Write},
process::{Command, Stdio},
sync::mpsc::{self, RecvTimeoutError},
time::{Duration, Instant},
};
use crate::config::{MemorySkillConfig, SecretDetectionMode};
use super::types::{EvidenceRef, MemorySkillAdvisory, MemorySkillCandidate, MemorySkillError};
const SECRET_DETECTOR_TIMEOUT: Duration = Duration::from_secs(10);
const SECRET_DETECTOR_THREAD_DRAIN_GRACE: Duration = Duration::from_millis(250);
pub fn scan_text(text: &str, config: &MemorySkillConfig) -> Result<(), MemorySkillError> {
scan_text_with_entropy(text, config, true)
}
fn scan_structured_text(text: &str, config: &MemorySkillConfig) -> Result<(), MemorySkillError> {
scan_text_with_entropy(text, config, false)
}
fn scan_text_with_entropy(
text: &str,
config: &MemorySkillConfig,
scan_entropy: bool,
) -> Result<(), MemorySkillError> {
let secret_scan_enabled =
config.redact_secrets || config.scan.secret_detection != SecretDetectionMode::Off;
if !secret_scan_enabled {
return Ok(());
}
let haystack = if config.scan.blocked_patterns_case_insensitive {
text.to_ascii_lowercase()
} else {
text.to_owned()
};
let normalized_haystack = normalize_assignment_separators(&haystack);
for pattern in &config.scan.effective_blocked_patterns() {
let needle = if config.scan.blocked_patterns_case_insensitive {
pattern.to_ascii_lowercase()
} else {
pattern.clone()
};
if !needle.is_empty()
&& (haystack.contains(&needle) || normalized_haystack.contains(&needle))
{
return Err(MemorySkillError::ScanRejected {
reason: format!("blocked pattern {pattern:?}"),
});
}
}
run_configured_secret_detector(text, config)?;
if scan_entropy {
for token in text.split(|character: char| {
!(character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '+' | '=' | '/'))
}) {
if token.len() >= 20 && shannon_entropy(token) >= config.scan.entropy_threshold {
return Err(MemorySkillError::ScanRejected {
reason: "high-entropy token".to_owned(),
});
}
}
}
Ok(())
}
pub(crate) fn scan_memory_skill_candidate(
candidate: &MemorySkillCandidate,
config: &MemorySkillConfig,
) -> Result<(), MemorySkillError> {
let mut prose = String::new();
let mut structured = String::new();
push_structured_field(
&mut structured,
"schema_version",
candidate.schema_version.to_string(),
);
push_structured_field(&mut structured, "id", &candidate.id);
push_structured_field(&mut structured, "fingerprint", &candidate.fingerprint);
push_structured_field(&mut structured, "learning_key", &candidate.learning_key);
push_structured_field(&mut structured, "status", candidate.status.to_string());
push_structured_field(
&mut structured,
"candidate_kind",
candidate.candidate_kind.as_str(),
);
push_structured_field(&mut structured, "scope", &candidate.scope);
push_structured_field(&mut structured, "source_commit", &candidate.source_commit);
push_structured_field(
&mut structured,
"truth_label",
candidate.truth_label.to_string(),
);
push_structured_field(
&mut structured,
"occurrence_count",
candidate.occurrence_count.to_string(),
);
push_structured_field(&mut structured, "slug", &candidate.slug);
push_structured_field(
&mut structured,
"created_at_unix",
candidate.created_at_unix.to_string(),
);
push_evidence_identifiers(
&mut structured,
"ledger_entry_ref",
&candidate.ledger_entry_ref,
);
push_evidence_identifiers(&mut structured, "source_run_ref", &candidate.source_run_ref);
push_evidence_identifier_fields(
&mut structured,
"occurrence_refs",
&candidate.occurrence_refs,
);
push_evidence_identifier_fields(&mut structured, "evidence", &candidate.evidence);
push_prose_field(&mut prose, "source_claim", &candidate.source_claim);
push_prose_field(&mut prose, "title", &candidate.title);
push_prose_field(&mut prose, "description", &candidate.description);
push_prose_fields(&mut prose, "when_to_use", &candidate.when_to_use);
push_prose_fields(&mut prose, "procedure_steps", &candidate.procedure_steps);
push_prose_fields(&mut prose, "pitfalls", &candidate.pitfalls);
push_prose_fields(
&mut prose,
"verification_steps",
&candidate.verification_steps,
);
push_evidence_summary(&mut prose, "ledger_entry_ref", &candidate.ledger_entry_ref);
push_evidence_summary(&mut prose, "source_run_ref", &candidate.source_run_ref);
push_evidence_summaries(&mut prose, "occurrence_refs", &candidate.occurrence_refs);
push_evidence_summaries(&mut prose, "evidence", &candidate.evidence);
scan_structured_text(&structured, config)?;
scan_text(&prose, config)
}
pub(crate) fn scan_memory_skill_advisory(
advisory: &MemorySkillAdvisory,
config: &MemorySkillConfig,
) -> Result<(), MemorySkillError> {
let mut prose = String::new();
let mut structured = String::new();
push_structured_field(
&mut structured,
"schema_version",
advisory.schema_version.to_string(),
);
push_structured_field(&mut structured, "id", &advisory.id);
push_structured_field(&mut structured, "fingerprint", &advisory.fingerprint);
push_structured_field(&mut structured, "dismissed", advisory.dismissed.to_string());
push_structured_field(
&mut structured,
"candidate_kind",
advisory.candidate_kind.as_str(),
);
push_structured_field(&mut structured, "slug", &advisory.slug);
push_structured_field(&mut structured, "source_commit", &advisory.source_commit);
push_structured_field(
&mut structured,
"truth_label",
advisory.truth_label.to_string(),
);
push_structured_field(
&mut structured,
"occurrence_count",
advisory.occurrence_count.to_string(),
);
push_structured_field(
&mut structured,
"created_at_unix",
advisory.created_at_unix.to_string(),
);
push_evidence_identifier_fields(
&mut structured,
"occurrence_refs",
&advisory.occurrence_refs,
);
push_prose_field(&mut prose, "title", &advisory.title);
push_prose_field(&mut prose, "reason", &advisory.reason);
push_evidence_summaries(&mut prose, "occurrence_refs", &advisory.occurrence_refs);
scan_structured_text(&structured, config)?;
scan_text(&prose, config)
}
fn push_structured_field(buffer: &mut String, field_name: &str, value: impl AsRef<str>) {
buffer.push_str(field_name);
buffer.push_str(": ");
buffer.push_str(value.as_ref());
buffer.push('\n');
}
fn push_evidence_identifier_fields(
buffer: &mut String,
field_name: &str,
references: &[EvidenceRef],
) {
for reference in references {
push_evidence_identifiers(buffer, field_name, reference);
}
}
fn push_evidence_identifiers(buffer: &mut String, field_name: &str, reference: &EvidenceRef) {
push_structured_field(buffer, field_name, &reference.kind);
if let Some(path) = &reference.path {
push_structured_field(buffer, field_name, path);
}
if let Some(selector) = &reference.selector {
push_structured_field(buffer, field_name, selector);
}
if let Some(run_id) = &reference.run_id {
push_structured_field(buffer, field_name, run_id);
}
if let Some(artifact) = &reference.artifact {
push_structured_field(buffer, field_name, artifact);
}
if let Some(memory_id) = &reference.memory_id {
push_structured_field(buffer, field_name, memory_id);
}
}
fn push_prose_fields(buffer: &mut String, field_name: &str, values: &[String]) {
for value in values {
push_prose_field(buffer, field_name, value);
}
}
fn push_evidence_summaries(buffer: &mut String, field_name: &str, references: &[EvidenceRef]) {
for reference in references {
push_evidence_summary(buffer, field_name, reference);
}
}
fn push_evidence_summary(buffer: &mut String, field_name: &str, reference: &EvidenceRef) {
if let Some(summary) = &reference.summary {
push_prose_field(buffer, field_name, summary);
}
}
fn push_prose_field(buffer: &mut String, field_name: &str, value: &str) {
buffer.push_str(field_name);
buffer.push_str(": ");
buffer.push_str(value);
buffer.push('\n');
}
fn normalize_assignment_separators(text: &str) -> String {
let mut output = String::with_capacity(text.len());
let mut pending_whitespace = String::new();
let mut previous_was_separator = false;
for character in text.chars() {
if character.is_whitespace() {
pending_whitespace.push(character);
continue;
}
if matches!(character, ':' | '=') {
output.push(character);
pending_whitespace.clear();
previous_was_separator = true;
continue;
}
if previous_was_separator {
pending_whitespace.clear();
} else {
output.push_str(&pending_whitespace);
}
pending_whitespace.clear();
output.push(character);
previous_was_separator = false;
}
output
}
fn run_configured_secret_detector(
text: &str,
config: &MemorySkillConfig,
) -> Result<(), MemorySkillError> {
let timeout = if config.scan.secret_detector_timeout_seconds == 0 {
SECRET_DETECTOR_TIMEOUT
} else {
Duration::from_secs(config.scan.secret_detector_timeout_seconds)
};
run_configured_secret_detector_with_timeout(text, config, timeout)
}
pub(crate) fn run_configured_secret_detector_with_timeout(
text: &str,
config: &MemorySkillConfig,
timeout: Duration,
) -> Result<(), MemorySkillError> {
let detector = config.scan.secret_detector.trim();
if detector.is_empty() {
return Ok(());
}
if !config.scan.allow_secret_detector_command {
return secret_detector_error(
config,
"configured detector command requires memory_skill.scan.allow_secret_detector_command = true"
.to_owned(),
);
}
let argv = match split_detector_command(detector) {
Ok(argv) if argv.is_empty() => return Ok(()),
Ok(argv) => argv,
Err(reason) => return secret_detector_error(config, reason),
};
let mut command = Command::new(&argv[0]);
command.args(&argv[1..]);
let mut child = match command
.stdin(Stdio::piped())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
{
Ok(child) => child,
Err(error) => return secret_detector_error(config, format!("failed to spawn: {error}")),
};
let (stderr_tx, stderr_rx) = mpsc::channel();
match child.stderr.take() {
Some(mut stderr) => std::thread::spawn(move || {
let mut buffer = Vec::new();
let _ = stderr_tx.send(stderr.read_to_end(&mut buffer).map(|_| buffer));
}),
None => return secret_detector_error(config, "missing stderr pipe".to_owned()),
};
let input = text.as_bytes().to_vec();
let (writer_tx, writer_rx) = mpsc::channel();
match child.stdin.take() {
Some(mut stdin) => std::thread::spawn(move || {
let _ = writer_tx.send(stdin.write_all(&input));
}),
None => return secret_detector_error(config, "missing stdin pipe".to_owned()),
};
let started_at = Instant::now();
let status = loop {
match child.try_wait() {
Ok(Some(status)) => break status,
Ok(None) if started_at.elapsed() >= timeout => {
let _ = child.kill();
let _ = child.wait();
return secret_detector_error(
config,
format!("timed out after {} seconds", timeout.as_secs()),
);
}
Ok(None) => std::thread::sleep(Duration::from_millis(10)),
Err(error) => {
return secret_detector_error(config, format!("failed to wait: {error}"));
}
}
};
let write_result = match receive_detector_thread(&writer_rx, "stdin writer") {
Ok(result) => result,
Err(reason) => return secret_detector_error(config, reason),
};
let stderr = match receive_detector_thread(&stderr_rx, "stderr reader") {
Ok(Ok(stderr)) => stderr,
Ok(Err(error)) => {
return secret_detector_error(config, format!("failed to read stderr: {error}"));
}
Err(reason) => return secret_detector_error(config, reason),
};
if !status.success() {
return Err(MemorySkillError::ScanRejected {
reason: format!(
"secret detector found blocked content: exited with status {:?} and {} bytes on stderr",
status.code(),
stderr.len()
),
});
}
if let Err(error) = write_result {
if matches!(
error.kind(),
io::ErrorKind::BrokenPipe | io::ErrorKind::WriteZero
) {
return Ok(());
}
return secret_detector_error(config, format!("failed to write stdin: {error}"));
}
Ok(())
}
fn receive_detector_thread<T>(receiver: &mpsc::Receiver<T>, label: &str) -> Result<T, String> {
match receiver.recv_timeout(SECRET_DETECTOR_THREAD_DRAIN_GRACE) {
Ok(result) => Ok(result),
Err(RecvTimeoutError::Timeout) => {
Err(format!("{label} did not finish after detector exit"))
}
Err(RecvTimeoutError::Disconnected) => Err(format!("{label} stopped without a result")),
}
}
fn secret_detector_error(
config: &MemorySkillConfig,
reason: String,
) -> Result<(), MemorySkillError> {
if config.scan.secret_detection == SecretDetectionMode::BestEffort {
tracing::warn!(reason = %reason, "memory-skill secret detector failed best-effort scan");
return Ok(());
}
Err(MemorySkillError::ScanRejected {
reason: format!("secret detector {reason}"),
})
}
fn split_detector_command(command: &str) -> Result<Vec<String>, String> {
let mut args = Vec::new();
let mut current = String::new();
let mut chars = command.chars();
let mut quote = None;
while let Some(character) = chars.next() {
match (character, quote) {
('\\', Some('"')) => {
if let Some(next) = chars.next() {
current.push(next);
}
}
('\\', None) => {
if let Some(next) = chars.next() {
current.push(next);
}
}
('"', None) => quote = Some('"'),
('\'', None) => quote = Some('\''),
('"', Some('"')) | ('\'', Some('\'')) => quote = None,
(character, None) if character.is_whitespace() => {
if !current.is_empty() {
args.push(std::mem::take(&mut current));
}
}
(character, _) => current.push(character),
}
}
if let Some(quote) = quote {
return Err(format!(
"secret detector command has unclosed {quote:?} quote"
));
}
if !current.is_empty() {
args.push(current);
}
Ok(args)
}
fn shannon_entropy(token: &str) -> f64 {
if token.is_empty() {
return 0.0;
}
let mut counts = BTreeMap::new();
for byte in token.bytes() {
*counts.entry(byte).or_insert(0usize) += 1;
}
let len = token.len() as f64;
counts
.values()
.map(|count| {
let probability = *count as f64 / len;
-probability * probability.log2()
})
.sum()
}