scour_secrets/lib.rs
1//! # scour-secrets
2//!
3//! Deterministic, one-way data sanitization engine.
4//!
5//! This crate provides the core replacement infrastructure for replacing
6//! sensitive values with category-aware, deterministic substitutes.
7//! Replacements are **one-way only** — there is no key file, mapping
8//! table, or restore mode. It is the foundation layer consumed by
9//! higher-level streaming and CLI components.
10//!
11//! ## Key Components
12//!
13//! - [`category::Category`] — Classification of sensitive values (email,
14//! IP, name, etc.) that determines replacement format.
15//! - [`generator::ReplacementGenerator`] — Trait abstracting replacement
16//! strategy (HMAC-deterministic or CSPRNG-random).
17//! - [`strategy::Strategy`] — Pluggable replacement strategies that can
18//! be called **directly** without any mapping table.
19//! - [`store::MappingStore`] — Optional thread-safe per-run dedup cache
20//! ensuring the same input always maps to the same output within a run.
21//! - [`scanner::StreamScanner`] — Streaming regex scanner with chunk +
22//! overlap for bounded-memory processing.
23//!
24//! ## Concurrency Model
25//!
26//! The `MappingStore` uses `DashMap` (shard-level locking) for the forward
27//! dedup cache. All types are `Send + Sync`.
28//!
29//! ## Example: Store-Level Replacement
30//!
31//! ```rust
32//! use scour_secrets::category::Category;
33//! use scour_secrets::generator::HmacGenerator;
34//! use scour_secrets::store::MappingStore;
35//! use std::sync::Arc;
36//!
37//! // Create a deterministic generator with a fixed seed.
38//! let generator = Arc::new(HmacGenerator::new([42u8; 32]));
39//!
40//! // Create the replacement store (optional capacity limit).
41//! let store = MappingStore::new(generator, None);
42//!
43//! // Sanitize a value (one-way).
44//! let sanitized = store.get_or_insert(&Category::Email, "alice@corp.com").unwrap();
45//! assert!(sanitized.contains("@corp.com"));
46//! assert_eq!(sanitized.len(), "alice@corp.com".len());
47//!
48//! // Same input → same output (per-run consistency).
49//! let again = store.get_or_insert(&Category::Email, "alice@corp.com").unwrap();
50//! assert_eq!(sanitized, again);
51//! ```
52//!
53//! ## Example: Streaming Scanner
54//!
55//! ```rust
56//! use scour_secrets::category::Category;
57//! use scour_secrets::generator::HmacGenerator;
58//! use scour_secrets::scanner::{ScanConfig, ScanPattern, StreamScanner};
59//! use scour_secrets::store::MappingStore;
60//! use std::sync::Arc;
61//!
62//! // Build patterns.
63//! let patterns = vec![
64//! ScanPattern::from_regex(r"alice@corp\.com", Category::Email, "alice_email").unwrap(),
65//! ];
66//!
67//! // Store with deterministic generator.
68//! let generator = Arc::new(HmacGenerator::new([42u8; 32]));
69//! let store = Arc::new(MappingStore::new(generator, Some(1_000_000)));
70//!
71//! // Scanner with default chunk config.
72//! let config = ScanConfig::new(1_048_576, 4096);
73//! let scanner = StreamScanner::new(patterns, store, config).unwrap();
74//!
75//! // Scan bytes in-memory.
76//! let input = b"Contact alice@corp.com for details.";
77//! let (output, stats) = scanner.scan_bytes(input).unwrap();
78//!
79//! assert_eq!(stats.replacements_applied, 1);
80//! assert_eq!(output.len(), input.len());
81//! ```
82//!
83//! ## Example: Log Context Extraction
84//!
85//! After sanitizing, scan the output for error/warning keywords and capture
86//! surrounding lines for LLM-friendly triage:
87//!
88//! ```rust
89//! use scour_secrets::log_context::{extract_context, LogContextConfig};
90//!
91//! let sanitized = "INFO request received\n\
92//! ERROR disk full on /dev/sda1\n\
93//! INFO retrying mount\n\
94//! WARN filesystem degraded\n\
95//! INFO recovery complete";
96//!
97//! let config = LogContextConfig::new().with_context_lines(1);
98//! let result = extract_context(sanitized, &config);
99//!
100//! // Two keyword hits: "error" and "warn".
101//! assert_eq!(result.match_count, 2);
102//!
103//! // First match: ERROR line with one line of context on each side.
104//! assert_eq!(result.matches[0].keyword, "error");
105//! assert_eq!(result.matches[0].before, vec!["INFO request received"]);
106//! assert_eq!(result.matches[0].after, vec!["INFO retrying mount"]);
107//! ```
108
109// Crate-level lint configuration.
110#![forbid(unsafe_code)]
111#![warn(clippy::all, clippy::pedantic)]
112// Allow specific pedantic lints that are too noisy for this crate.
113#![allow(
114 clippy::module_name_repetitions,
115 clippy::missing_panics_doc,
116 clippy::must_use_candidate, // We add #[must_use] manually on key APIs.
117 clippy::uninlined_format_args,
118 clippy::redundant_closure_for_method_calls,
119 clippy::doc_markdown,
120 clippy::similar_names
121)]
122
123pub mod allowlist;
124pub mod atomic;
125pub mod category;
126pub mod entropy;
127pub mod error;
128pub mod generator;
129pub mod llm;
130pub mod log_context;
131pub mod processor;
132pub mod report;
133pub mod scanner;
134pub mod secrets;
135pub mod store;
136pub mod strategy;
137pub mod strip_values;
138
139// Re-exports for convenience.
140pub use atomic::{atomic_write, atomic_write_private, AtomicFileWriter};
141pub use category::Category;
142pub use entropy::{entropy_scan_bytes, merge_entropy_counts, EntropyCharset, EntropyConfig};
143pub use error::{Result, SanitizeError};
144pub use generator::{HmacGenerator, LengthPolicy, RandomGenerator, ReplacementGenerator};
145pub use llm::{
146 format_llm_prompt, format_llm_prompt_reference, resolve_llm_template, LlmEntry, LlmPathEntry,
147 PROMPT_PREAMBLE, TEMPLATE_REVIEW_CONFIG, TEMPLATE_REVIEW_SECURITY, TEMPLATE_TROUBLESHOOT,
148};
149pub use log_context::{
150 extract_context, extract_context_reader, LogContextConfig, LogContextMatch, LogContextResult,
151 DEFAULT_CONTEXT_LINES, DEFAULT_KEYWORDS, DEFAULT_MAX_MATCHES,
152};
153#[cfg(feature = "archive")]
154pub use processor::archive::{
155 ArchiveFilter, ArchiveFormat, ArchiveProcessor, ArchiveProgress, ArchiveStats, EntryCallback,
156};
157pub use processor::limits::DEFAULT_ARCHIVE_DEPTH;
158pub use processor::{
159 FieldNameSignal, FieldRule, FileTypeProfile, Processor, ProcessorRegistry, Replacement,
160 DEFAULT_FIELD_SIGNAL_THRESHOLD,
161};
162pub use report::{
163 FileReport, MatchLocationsResult, ReportBuilder, ReportMetadata, ReportSummary, SanitizeReport,
164};
165pub use scanner::{
166 MatchLocation, ScanConfig, ScanPattern, ScanProgress, ScanStats, SecretsLoadResult,
167 StreamScanner, MIN_DISCOVERED_LITERAL_LEN,
168};
169pub use secrets::{
170 decrypt_secrets, encrypt_secrets, entries_to_patterns, extract_allow_patterns,
171 load_encrypted_secrets, load_plaintext_secrets, load_secrets_auto, looks_encrypted,
172 parse_category, parse_secrets, serialize_secrets, AutoLoadedSecrets, PatternCompileResult,
173 SecretEntry, SecretsFormat,
174};
175pub use store::{MappingStore, StoreSnapshot};
176pub use strategy::{
177 CategoryAwareStrategy, EntropyMode, FakeIp, HmacHash, PreserveLength, RandomString, RandomUuid,
178 Strategy, StrategyGenerator,
179};
180pub use strip_values::strip_values_from_text;