second-brain-core 0.13.5

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::sync::LazyLock;

use regex::Regex;

use super::{Finding, SecretKind, has_random_run, marker, record};

static PEM_BEGIN: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----").expect("pattern compiles")
});
static PEM_END_AT_START: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(r"\A-----END [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----").expect("pattern compiles")
});

/// A body without a base64 run this long holds no key, so the header is a
/// string in code or prose and is kept.
const MIN_KEY_RUN: usize = 32;

pub(super) fn redact_private_keys(text: &str, findings: &mut Vec<Finding>) -> Option<String> {
    let pem = redact_pem(text, findings);
    redact_putty(pem.as_deref().unwrap_or(text), findings).or(pem)
}

/// A block runs from its BEGIN line to an END line when only key material lies
/// between them; otherwise (a truncated transcript) over the armor headers and
/// base64 lines that follow. Every scan stops at the next run of five dashes,
/// so no byte is scanned for two headers and the whole pass stays linear.
fn redact_pem(text: &str, findings: &mut Vec<Finding>) -> Option<String> {
    let mut out = String::new();
    let mut last = 0;
    let mut count = 0;
    let mut cursor = 0;
    while let Some(begin) = PEM_BEGIN.find_at(text, cursor) {
        let rest = &text[begin.end()..];
        let Some(len) = footer_block_len(rest).or_else(|| bare_key_len(rest)) else {
            cursor = begin.end();
            continue;
        };
        out.push_str(&text[last..begin.start()]);
        out.push_str(&marker(SecretKind::PrivateKey));
        last = begin.end() + len;
        cursor = last;
        count += 1;
    }
    if count == 0 {
        return None;
    }
    out.push_str(&text[last..]);
    record(findings, SecretKind::PrivateKey, count);
    Some(out)
}

static PUTTY_PRIVATE_LINES: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"\bPrivate-Lines: ([0-9]{1,3})\r?\n").expect("pattern compiles"));

/// The base64 lines a PuTTY key file's `Private-Lines: N` header counts, each
/// possibly after a Read tool's line number; the first line's number is kept.
fn redact_putty(text: &str, findings: &mut Vec<Finding>) -> Option<String> {
    let bytes = text.as_bytes();
    let mut out = String::new();
    let mut last = 0;
    let mut count = 0;
    for caps in PUTTY_PRIVATE_LINES.captures_iter(text) {
        let header_end = caps.get(0).expect("group 0 always matches").end();
        let lines: usize = caps[1].parse().expect("one to three digits");
        let start = past_numbered_indent(bytes, header_end);
        let (mut end, mut pos) = (start, start);
        for _ in 0..lines {
            pos = past_numbered_indent(bytes, pos);
            let len = bytes[pos..].iter().take_while(|&&b| is_base64(b)).count();
            let rest = &bytes[pos + len..];
            let newline = if rest.starts_with(b"\r\n") {
                2
            } else {
                usize::from(rest.starts_with(b"\n"))
            };
            if len == 0 || (newline == 0 && !rest.is_empty()) {
                break;
            }
            end = pos + len;
            if newline == 0 {
                break;
            }
            pos = end + newline;
        }
        if end == start {
            continue;
        }
        out.push_str(&text[last..start]);
        out.push_str(&marker(SecretKind::PrivateKey));
        last = end;
        count += 1;
    }
    if count == 0 {
        return None;
    }
    out.push_str(&text[last..]);
    record(findings, SecretKind::PrivateKey, count);
    Some(out)
}

/// Past the spaces and line number a Read tool puts before a line at `i`, or
/// `i` when no line number follows the spaces.
fn past_numbered_indent(bytes: &[u8], i: usize) -> usize {
    let spaces = bytes[i..].iter().take_while(|&&b| b == b' ').count();
    let numbered = skip_line_number(bytes, i + spaces);
    if numbered > i + spaces { numbered } else { i }
}

fn is_base64(b: u8) -> bool {
    b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')
}

/// Quotes, whitespace, string concatenation and escaped newlines around key
/// lines pasted as code literals or indented in config.
fn is_key_glue(b: u8) -> bool {
    matches!(
        b,
        b' ' | b'\t' | b'\r' | b'\n' | b'"' | b'\'' | b'`' | b',' | b'\\' | b'>' | b'|'
    )
}

/// A base64 run long enough to be key material, mixing upper and lower case
/// and digits as encoded random bytes do; digests and paths do not.
fn has_key_run(s: &[u8]) -> bool {
    has_random_run(s, MIN_KEY_RUN, is_base64)
}

/// End of an END line starting at `i`, past a diff marker dash before it.
fn footer_end(s: &str, i: usize) -> Option<usize> {
    let dashes = s.as_bytes()[i..].iter().take_while(|&&b| b == b'-').count();
    let start = i + dashes.saturating_sub(5);
    PEM_END_AT_START
        .find(&s[start..])
        .map(|footer| start + footer.end())
}

/// Length through the END line, when only base64, armor and glue lie between.
fn footer_block_len(s: &str) -> Option<usize> {
    let bytes = s.as_bytes();
    let dashes = bytes.windows(5).position(|w| w == b"-----")?;
    let between = &bytes[..dashes];
    let only_material = between
        .iter()
        .all(|&b| is_base64(b) || is_key_glue(b) || is_line_prefix_byte(b));
    if !only_material || !has_key_run(between) {
        return None;
    }
    footer_end(s, dashes)
}

/// Length of a key body with no END line in reach, extended over an END line
/// that directly follows it.
fn bare_key_len(s: &str) -> Option<usize> {
    let body = key_body_len(s);
    if !has_key_run(&s.as_bytes()[..body]) {
        return None;
    }
    let gap = s.as_bytes()[body..]
        .iter()
        .take_while(|&&b| is_key_glue(b) || b == b'n' || b == b'r')
        .count();
    Some(footer_end(s, body + gap).unwrap_or(body))
}

/// Diff markers, comment markers (`#`, `//`, `*`), PHP's `.` concatenation
/// and the `→` after a Read tool's line number, which may start key lines.
fn is_line_prefix_byte(b: u8) -> bool {
    matches!(b, b':' | b'-' | b'#' | b'*' | b'.') || "\u{2192}".as_bytes().contains(&b)
}

/// Past a Read tool's line number (`   12\t` or `   12\u{2192}`) at `i`.
fn skip_line_number(bytes: &[u8], i: usize) -> usize {
    let digits = bytes[i..].iter().take_while(|b| b.is_ascii_digit()).count();
    let rest = &bytes[i + digits..];
    if digits == 0 {
        i
    } else if rest.starts_with(b"\t") {
        i + digits + 1
    } else if rest.starts_with("\u{2192}".as_bytes()) {
        i + digits + "\u{2192}".len()
    } else {
        i
    }
}

const ARMOR_HEADERS: [&[u8]; 6] = [
    b"Version: ",
    b"Comment: ",
    b"Proc-Type: ",
    b"DEK-Info: ",
    b"Hash: ",
    b"Charset: ",
];

/// A line of several space-separated tokens, or one not ended by a line
/// break, keeps only tokens from this length, so prose after a truncated key
/// is kept.
const MIN_KEY_TOKEN: usize = 16;

fn line_break_len(bytes: &[u8], i: usize) -> Option<usize> {
    match bytes.get(i) {
        None => Some(0),
        Some(b'\n') => Some(1),
        Some(b'\\') if bytes.get(i + 1) == Some(&b'n') => Some(2),
        Some(b'\\') if bytes[i + 1..].starts_with(b"r\\n") => Some(4),
        Some(_) => None,
    }
}

/// Includes PHP's `.` between a key line's string and the next.
fn is_inline_glue(b: u8) -> bool {
    matches!(b, b' ' | b'\t' | b'\r' | b'"' | b'\'' | b',' | b'.')
}

struct KeyLine {
    stop: usize,
    kept_end: Option<usize>,
    armor: bool,
    clean: bool,
    rejected_token: bool,
}

/// One line of a key body from `i`: indentation, quotes, a Read tool's line
/// number and a diff, quote or comment marker, then either an armor header
/// or base64 tokens split by spaces.
/// Never scans past `-----`.
fn scan_key_line(bytes: &[u8], mut i: usize, armor_allowed: bool) -> KeyLine {
    let skip_glue = |mut i: usize| {
        while bytes.get(i).copied().is_some_and(is_inline_glue) {
            i += 1;
        }
        i
    };
    i = skip_glue(i);
    let numbered = skip_line_number(bytes, i);
    if numbered > i {
        i = skip_glue(numbered);
    }
    if bytes[i..].starts_with(b"// ") {
        i = skip_glue(i + 2);
    } else if matches!(bytes.get(i), Some(b'-' | b'>' | b'|' | b'#' | b'*'))
        && !bytes[i..].starts_with(b"-----")
    {
        i = skip_glue(i + 1);
    }
    if armor_allowed && ARMOR_HEADERS.iter().any(|h| bytes[i..].starts_with(h)) {
        while line_break_len(bytes, i).is_none() && !bytes[i..].starts_with(b"-----") {
            i += 1;
        }
        return KeyLine {
            stop: i,
            kept_end: None,
            armor: true,
            clean: line_break_len(bytes, i).is_some(),
            rejected_token: false,
        };
    }
    let mut last_token_end = None;
    let mut long_tokens_end = None;
    let mut tokens = 0;
    let mut all_long = true;
    loop {
        let start = i;
        while bytes.get(i).copied().is_some_and(is_base64) {
            i += 1;
        }
        let token = &bytes[start..i];
        if token.iter().any(|&b| b != b'+') {
            last_token_end = Some(i);
            tokens += 1;
            all_long &= token.len() >= MIN_KEY_TOKEN;
            if all_long {
                long_tokens_end = Some(i);
            }
        }
        let glue = i;
        while bytes.get(i).copied().is_some_and(is_inline_glue) {
            i += 1;
        }
        if i == glue {
            break;
        }
    }
    let clean = line_break_len(bytes, i).is_some();
    let kept_end = if clean && tokens == 1 {
        last_token_end
    } else {
        long_tokens_end
    };
    KeyLine {
        stop: i,
        kept_end,
        armor: false,
        clean,
        rejected_token: kept_end != last_token_end,
    }
}

/// Length of the key body after a header: armor header lines, one blank line
/// after them, then base64 lines, each ended by a newline or a literal `\n`.
fn key_body_len(s: &str) -> usize {
    let bytes = s.as_bytes();
    let mut end = 0;
    let mut i = 0;
    let mut first = true;
    let mut after_armor = false;
    loop {
        let line = scan_key_line(bytes, i, !first);
        if let Some(kept) = line.kept_end {
            end = kept;
        }
        let empty = line.stop == i;
        if !line.clean || line.rejected_token || (empty && !first && !after_armor) {
            break;
        }
        let step = line_break_len(bytes, line.stop).unwrap_or(0);
        if step == 0 {
            break;
        }
        after_armor = line.armor;
        first = false;
        i = line.stop + step;
    }
    end
}