keyhog_scanner/decode/
mod.rs

1//! Decode-through scanning: decode encoded strings before pattern matching.
2//!
3//! Catches secrets hidden behind encoding layers - Kubernetes manifests,
4//! CI/CD configs, URL-escaped payloads, string escapes, and hex-encoded
5//! credentials.
6
7mod base64;
8pub(crate) mod caesar;
9pub(crate) mod hex;
10pub(crate) mod inflate;
11#[cfg(feature = "decode")]
12mod javascript_static;
13mod json;
14mod limits;
15mod pipeline;
16pub(crate) mod policy;
17pub(crate) mod reverse;
18mod unicode_escape;
19mod url;
20pub(crate) mod util;
21
22pub use base64::{base64_decode, find_base64_strings, z85_decode};
23// `is_base64_candidate_byte` is the single canonical base64/url-safe alphabet
24// predicate; it is `pub` (not `pub(crate)`) because `keyhog-cli`'s autoroute
25// decode-density scanner (`orchestrator::dispatch::backend::workload`) is a
26// cross-crate consumer that must route through this one owner rather than
27// re-inline the byte set. The remaining three stay crate-internal.
28pub use base64::is_base64_candidate_byte;
29pub(crate) use base64::{
30    contains_non_padding_equals, is_standard_base64_byte, standard_base64_shape,
31};
32pub use hex::{find_hex_strings, hex_decode};
33pub(crate) use pipeline::CompiledDecoderPlan;
34pub(crate) use pipeline::{
35    bytecount_newlines, decoder_profile_dump, decoder_profile_reset, extract_profile_dump,
36    extract_profile_reset, splice_decoded_payload_at, with_extracted_value_spans,
37};
38#[cfg(feature = "decode")]
39pub(crate) use pipeline::{decoder_admission, default_decoder_names};
40pub use pipeline::{register_decoder, try_register_decoder, DecoderRegistrationError};
41#[cfg(test)]
42pub(crate) use pipeline::{register_thread_decoder, ScopedDecoderRegistration};
43pub(crate) use util::take_hex_digits;
44
45use keyhog_core::Chunk;
46
47#[cfg(feature = "decode")]
48pub(crate) fn decode_chunk_with_policy(
49    chunk: &Chunk,
50    policy: &policy::CompiledDecodeTransformPolicy,
51    decoder_plan: &CompiledDecoderPlan,
52    max_depth: usize,
53    validate: bool,
54    deadline: Option<std::time::Instant>,
55    screen: Option<&crate::alphabet_filter::AlphabetScreen>,
56) -> Vec<Chunk> {
57    pipeline::decode_chunk_with_policy(
58        chunk,
59        policy,
60        decoder_plan,
61        max_depth,
62        validate,
63        deadline,
64        screen,
65    )
66}
67
68/// Direct primitive compatibility for the public testing facade. Product
69/// scans always call `decode_chunk_with_policy` with their active detector
70/// corpus.
71pub(crate) fn decode_chunk(
72    chunk: &Chunk,
73    max_depth: usize,
74    validate: bool,
75    deadline: Option<std::time::Instant>,
76    screen: Option<&crate::alphabet_filter::AlphabetScreen>,
77) -> Vec<Chunk> {
78    pipeline::decode_chunk_with_active_decoders(
79        chunk,
80        policy::bundled_compat_policy(),
81        max_depth,
82        validate,
83        deadline,
84        screen,
85    )
86}
87
88pub(crate) fn unicode_escape_decode(input: &str) -> Result<String, ()> {
89    unicode_escape::unicode_escape_decode(input)
90}
91
92#[cfg(feature = "decode")]
93pub(crate) fn quoted_printable_decode(input: &str) -> Result<String, ()> {
94    url::quoted_printable_decode(input)
95}
96
97#[cfg(feature = "decode")]
98pub(crate) fn mime_encoded_word_decode(input: &str) -> Result<String, ()> {
99    url::mime_encoded_word_decode(input)
100}
101
102#[cfg(feature = "decode")]
103pub(crate) fn octal_escape_decode(input: &str) -> Result<String, ()> {
104    url::octal_escape_decode(input)
105}
106
107pub(crate) fn extracted_value_strings_for_test(text: &str) -> Vec<String> {
108    pipeline::with_extracted_value_spans(text, |values| {
109        values.iter().map(|value| value.value.clone()).collect()
110    })
111}
112
113#[cfg(feature = "decode")]
114fn valid_html_numeric_entity_len(data: &[u8]) -> Option<usize> {
115    if !data.starts_with(b"&#") {
116        return None;
117    }
118
119    let mut index = 2usize;
120    let radix = if matches!(data.get(index), Some(b'x' | b'X')) {
121        index += 1;
122        16u32
123    } else {
124        10u32
125    };
126    let digits_start = index;
127    let mut codepoint = 0u32;
128    while index < data.len() && index - digits_start < url::MAX_NUMERIC_ENTITY_DIGITS {
129        let digit = match data[index] {
130            b'0'..=b'9' => u32::from(data[index] - b'0'),
131            b'a'..=b'f' if radix == 16 => u32::from(data[index] - b'a') + 10,
132            b'A'..=b'F' if radix == 16 => u32::from(data[index] - b'A') + 10,
133            _ => break,
134        };
135        codepoint = codepoint.checked_mul(radix)?.checked_add(digit)?;
136        index += 1;
137    }
138
139    if index == digits_start || data.get(index) != Some(&b';') {
140        return None;
141    }
142    char::from_u32(codepoint)?;
143    Some(index + 1)
144}
145
146/// Cheap O(n), allocation-free gate: does `data` contain an encoded shape long
147/// enough to plausibly hide a credential?
148///
149/// The direct-match prefilters (`AlphabetScreen`, the bigram bloom) reject a
150/// chunk that carries none of any detector's literal bytes/bigrams - which is
151/// EXACTLY the shape of a fully-encoded secret, whose plaintext keyword/prefix
152/// only appears AFTER decoding. Those chunks would be dropped before
153/// decode-through ever ran. This gate lets the scan entry route such a chunk
154/// into a decode-only pass instead of skipping it, bounded to chunks that
155/// actually look encoded so normal traffic keeps the fast skip.
156#[cfg(feature = "decode")]
157pub(crate) fn has_decodable_payload(data: &[u8]) -> bool {
158    // Static XOR programs can consist entirely of short decimal literals, so
159    // they do not necessarily contain the long base64/hex run recognized by
160    // the byte-density loop below. Without this marker pair the SIMD entry
161    // path can skip decode post-processing while CPU fallback runs it, causing
162    // backend-dependent recall. The full bounded grammar still validates the
163    // source in `javascript_static`; this is admission only.
164    let mut run = 0usize;
165    let mut percent_escapes = 0usize;
166    let mut backslash_escapes = 0usize;
167    let mut html_numeric_entities = 0usize;
168    let mut has_from_char_code = false;
169    let mut has_xor_operator = false;
170    let mut i = 0usize;
171
172    while i < data.len() {
173        let b = data[i];
174
175        if b == b'^' {
176            has_xor_operator = true;
177            if has_from_char_code {
178                return true;
179            }
180        } else if b == b'f' && data[i..].starts_with(b"fromCharCode") {
181            has_from_char_code = true;
182            if has_xor_operator {
183                return true;
184            }
185        }
186
187        if b == b'%'
188            && i + 2 < data.len()
189            && data[i + 1].is_ascii_hexdigit()
190            && data[i + 2].is_ascii_hexdigit()
191        {
192            percent_escapes += 1;
193            if percent_escapes >= limits::MIN_PERCENT_ESCAPES {
194                return true;
195            }
196            run = 0;
197            i += 3;
198            continue;
199        }
200
201        if b == b'&' {
202            if let Some(entity_len) = valid_html_numeric_entity_len(&data[i..]) {
203                html_numeric_entities += 1;
204                if html_numeric_entities >= limits::MIN_HTML_NUMERIC_ENTITIES {
205                    return true;
206                }
207                run = 0;
208                i += entity_len;
209                continue;
210            }
211        }
212
213        if b == b'\\' && i + 1 < data.len() {
214            match data[i + 1] {
215                b'u' if i + 5 < data.len()
216                    && data[i + 2..i + 6]
217                        .iter()
218                        .all(|digit| digit.is_ascii_hexdigit()) =>
219                {
220                    backslash_escapes += 1;
221                    if backslash_escapes >= limits::MIN_BACKSLASH_ESCAPES {
222                        return true;
223                    }
224                    run = 0;
225                    i += 6;
226                    continue;
227                }
228                b'x' if i + 3 < data.len()
229                    && data[i + 2..i + 4]
230                        .iter()
231                        .all(|digit| digit.is_ascii_hexdigit()) =>
232                {
233                    backslash_escapes += 1;
234                    if backslash_escapes >= limits::MIN_BACKSLASH_ESCAPES {
235                        return true;
236                    }
237                    run = 0;
238                    i += 4;
239                    continue;
240                }
241                // C-style octal escape `\NNN` (exactly 3 octal digits), the
242                // trigger grammar of `OctalEscapeDecoder::contains_octal_escape`.
243                // Without this arm the octal digits between the backslashes form
244                // runs of only 3 (well under MIN_DECODABLE_RUN=24) and no other
245                // arm matches, so an octal-ONLY chunk returned false here and the
246                // whole decode pipeline was skipped, leaving the registered
247                // octal decoder unreachable for octal-encoded payloads (a silent
248                // recall hole, Law 10). Counts toward the same backslash-escape
249                // threshold as `\u`/`\x` so octal reaches detection parity with
250                // its sibling escapes.
251                b'0'..=b'7'
252                    if i + 3 < data.len()
253                        && (b'0'..=b'7').contains(&data[i + 2])
254                        && (b'0'..=b'7').contains(&data[i + 3]) =>
255                {
256                    backslash_escapes += 1;
257                    if backslash_escapes >= limits::MIN_BACKSLASH_ESCAPES {
258                        return true;
259                    }
260                    run = 0;
261                    i += 4;
262                    continue;
263                }
264                _ => {}
265            }
266        }
267
268        // base64 (standard + url-safe) and hex share this alphabet; padding
269        // `=` is included so a trailing-padded blob still counts.
270        if is_base64_candidate_byte(b) {
271            run += 1;
272            if run >= limits::MIN_DECODABLE_RUN {
273                return true;
274            }
275        } else {
276            run = 0;
277        }
278        i += 1;
279    }
280    false
281}
282
283/// A trait for decoding chunks to find hidden secrets.
284pub trait Decoder: Send + Sync {
285    fn name(&self) -> &'static str;
286
287    /// Stable implementation version used by compiled scanner and autoroute
288    /// identity. Increment this value whenever the decoder can emit a different
289    /// set of chunks for the same input.
290    fn version(&self) -> &'static str {
291        "1"
292    }
293
294    /// Bounded work projection for this decoder on `chunk`.
295    ///
296    /// Custom decoders default to an unknown, conservative sketch. Built-in
297    /// decoders override this beside the grammar used by `decode_chunk`.
298    fn admission_sketch(&self, _chunk: &Chunk) -> DecodeAdmissionSketch {
299        DecodeAdmissionSketch::UNKNOWN
300    }
301
302    /// Whether this decoder can produce output for `chunk`.
303    ///
304    /// Custom decoders default to [`DecodeAdmission::Unknown`], which always
305    /// fails open. Built-in decoders derive this from the sketch owned next to
306    /// the grammar used by [`Self::decode_chunk`]. Only `Impossible` permits the
307    /// engine to skip decode post-processing.
308    fn admission(&self, _chunk: &Chunk) -> DecodeAdmission {
309        self.admission_sketch(_chunk).admission()
310    }
311
312    fn decode_chunk(&self, chunk: &Chunk) -> Vec<Chunk>;
313}
314
315/// Bounded, content-free projection of decoder work.
316///
317/// The sketch contains only decoder-mechanism bits and saturating cost counters.
318/// It carries no source bytes, offsets, values, or content-derived hashes, so
319/// it is safe to persist as part of autoroute workload identity.
320#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
321pub struct DecodeAdmissionSketch {
322    kind_mask: u32,
323    candidate_count: u16,
324    candidate_bytes: u32,
325    unknown: bool,
326}
327
328impl DecodeAdmissionSketch {
329    pub const BASE64: u32 = 1 << 0;
330    pub const HEX: u32 = 1 << 1;
331    pub const URL: u32 = 1 << 2;
332    pub const QUOTED_PRINTABLE: u32 = 1 << 3;
333    pub const HTML_NAMED_ENTITY: u32 = 1 << 4;
334    pub const HTML_NUMERIC_ENTITY: u32 = 1 << 5;
335    pub const OCTAL_ESCAPE: u32 = 1 << 6;
336    pub const MIME_ENCODED_WORD: u32 = 1 << 7;
337    pub const JSON: u32 = 1 << 8;
338    pub const UNICODE_ESCAPE: u32 = 1 << 9;
339    pub const Z85: u32 = 1 << 10;
340    pub const JAVASCRIPT_STATIC: u32 = 1 << 11;
341    pub const REVERSE: u32 = 1 << 12;
342    pub const CAESAR: u32 = 1 << 13;
343    /// Bounded gzip/zlib inflation reached through the base64 decoder.
344    pub const COMPRESSED_CONTAINER: u32 = 1 << 14;
345
346    pub const NONE: Self = Self {
347        kind_mask: 0,
348        candidate_count: 0,
349        candidate_bytes: 0,
350        unknown: false,
351    };
352
353    pub const UNKNOWN: Self = Self {
354        kind_mask: 0,
355        candidate_count: u16::MAX,
356        candidate_bytes: u32::MAX,
357        unknown: true,
358    };
359
360    pub const fn kind_mask(self) -> u32 {
361        self.kind_mask
362    }
363
364    pub const fn candidate_count(self) -> u16 {
365        self.candidate_count
366    }
367
368    pub const fn candidate_bytes(self) -> u32 {
369        self.candidate_bytes
370    }
371
372    pub const fn has_unknown(self) -> bool {
373        self.unknown
374    }
375
376    pub fn merge(&mut self, other: Self) {
377        self.kind_mask |= other.kind_mask;
378        self.candidate_count = self.candidate_count.saturating_add(other.candidate_count);
379        self.candidate_bytes = self.candidate_bytes.saturating_add(other.candidate_bytes);
380        self.unknown |= other.unknown;
381        if self.unknown {
382            self.candidate_count = u16::MAX;
383            self.candidate_bytes = u32::MAX;
384        }
385    }
386
387    pub(crate) fn possible(kind: u32, candidate_count: usize, candidate_bytes: usize) -> Self {
388        Self {
389            kind_mask: kind,
390            candidate_count: candidate_count.min(u16::MAX as usize) as u16,
391            candidate_bytes: candidate_bytes.min(u32::MAX as usize) as u32,
392            unknown: false,
393        }
394    }
395
396    pub(crate) const fn admission(self) -> DecodeAdmission {
397        if self.unknown {
398            DecodeAdmission::Unknown
399        } else if self.kind_mask == 0 {
400            DecodeAdmission::Impossible
401        } else {
402            DecodeAdmission::Possible
403        }
404    }
405}
406
407/// Effective immutable decode policy captured from one compiled scanner.
408///
409/// Autoroute keeps this value with its router so workload classification uses
410/// the same decode enablement and input ceiling as the scanner it will run.
411#[derive(Clone, Debug)]
412pub struct DecodeWorkloadPlan {
413    enabled: bool,
414    max_input_bytes: usize,
415    transforms: DecodeTransformPolicyHandle,
416    decoders: DecoderPlanHandle,
417}
418
419#[derive(Clone, Debug)]
420enum DecodeTransformPolicyHandle {
421    Bundled,
422    Compiled(std::sync::Arc<policy::CompiledDecodeTransformPolicy>),
423}
424
425#[derive(Clone, Debug)]
426enum DecoderPlanHandle {
427    Active,
428    Compiled(std::sync::Arc<CompiledDecoderPlan>),
429}
430
431impl DecodeTransformPolicyHandle {
432    fn policy(&self) -> &policy::CompiledDecodeTransformPolicy {
433        match self {
434            Self::Bundled => policy::bundled_compat_policy(),
435            Self::Compiled(policy) => policy,
436        }
437    }
438}
439
440impl PartialEq for DecodeWorkloadPlan {
441    fn eq(&self, other: &Self) -> bool {
442        self.enabled == other.enabled
443            && self.max_input_bytes == other.max_input_bytes
444            && self.transforms.policy().identity() == other.transforms.policy().identity()
445            && match (&self.decoders, &other.decoders) {
446                (DecoderPlanHandle::Active, DecoderPlanHandle::Active) => true,
447                (DecoderPlanHandle::Compiled(left), DecoderPlanHandle::Compiled(right)) => {
448                    left.identity() == right.identity()
449                }
450                _ => false,
451            }
452    }
453}
454
455impl Eq for DecodeWorkloadPlan {}
456
457impl DecodeWorkloadPlan {
458    /// Resolve decode enablement from the same depth and byte limits consumed
459    /// by [`crate::ScannerConfig`]. A zero depth disables the mechanism. This
460    /// standalone constructor uses the bundled compatibility prefix policy;
461    /// [`crate::CompiledScanner::decode_workload_plan`] carries its exact active
462    /// detector policy instead.
463    pub const fn from_limits(max_depth: usize, max_input_bytes: usize) -> Self {
464        Self {
465            enabled: cfg!(feature = "decode") && max_depth > 0,
466            max_input_bytes,
467            transforms: DecodeTransformPolicyHandle::Bundled,
468            decoders: DecoderPlanHandle::Active,
469        }
470    }
471
472    pub(crate) fn from_compiled_limits(
473        max_depth: usize,
474        max_input_bytes: usize,
475        transforms: std::sync::Arc<policy::CompiledDecodeTransformPolicy>,
476        decoders: std::sync::Arc<CompiledDecoderPlan>,
477    ) -> Self {
478        Self {
479            enabled: cfg!(feature = "decode") && max_depth > 0,
480            max_input_bytes,
481            transforms: DecodeTransformPolicyHandle::Compiled(transforms),
482            decoders: DecoderPlanHandle::Compiled(decoders),
483        }
484    }
485
486    pub const fn enabled(&self) -> bool {
487        self.enabled
488    }
489
490    pub const fn max_input_bytes(&self) -> usize {
491        self.max_input_bytes
492    }
493
494    pub fn admits(&self, chunk: &Chunk) -> bool {
495        self.enabled && chunk.data.len() <= self.max_input_bytes
496    }
497
498    /// Project work only when the compiled scanner can execute decode-through
499    /// for this exact chunk.
500    pub fn sketch(&self, chunk: &Chunk) -> DecodeAdmissionSketch {
501        #[cfg(not(feature = "decode"))]
502        {
503            // LAW10: no runtime effect; a decode-disabled build has no compiled decoder whose findings this binding could affect.
504            let _ = chunk;
505            return DecodeAdmissionSketch::NONE;
506        }
507        #[cfg(feature = "decode")]
508        if !self.admits(chunk) {
509            DecodeAdmissionSketch::NONE
510        } else {
511            match &self.decoders {
512                DecoderPlanHandle::Active => {
513                    pipeline::active_decoder_admission_sketch(chunk, self.transforms.policy())
514                }
515                DecoderPlanHandle::Compiled(plan) => {
516                    pipeline::decoder_admission_sketch(chunk, self.transforms.policy(), plan)
517                }
518            }
519        }
520    }
521}
522
523/// Compute a standalone decode work sketch with the bundled compatibility
524/// prefix policy. Autoroute uses [`crate::CompiledScanner::decode_workload_plan`]
525/// so its sketch matches the active detector corpus.
526#[cfg(feature = "decode")]
527pub fn decode_admission_sketch(chunk: &Chunk) -> DecodeAdmissionSketch {
528    pipeline::active_decoder_admission_sketch(chunk, policy::bundled_compat_policy())
529}
530
531/// Decode-disabled builds contribute no decoder work to autoroute identity.
532#[cfg(not(feature = "decode"))]
533pub fn decode_admission_sketch(_chunk: &Chunk) -> DecodeAdmissionSketch {
534    DecodeAdmissionSketch::NONE
535}
536
537/// Proof carried from decoder-owned grammars to the scan admission path.
538#[derive(Clone, Copy, Debug, Eq, PartialEq)]
539#[non_exhaustive]
540pub enum DecodeAdmission {
541    /// The decoder does not expose a complete admission predicate. Fail open.
542    Unknown,
543    /// The decoder grammar can produce at least one output candidate.
544    Possible,
545    /// The decoder grammar proves that it cannot produce output.
546    Impossible,
547}
548
549/// Candidate encoded string discovered during pre-decoding extraction.
550pub struct EncodedString {
551    pub value: String,
552}
keyhog_scanner/decode/mod.rs

keyhog_scanner/decode/
mod.rs
