khive_runtime/

secret_gate.rs

//! Write-time secret detection gate.
//!
//! Scans caller-supplied content strings before any storage write. A match
//! causes a hard `RuntimeError::SecretDetected` that names the detector and
//! carries a masked excerpt internally. Its display names the rule and trigger
//! without echoing any candidate text.
//!
//! Scope: **credentials only** — API keys, tokens, private keys, passwords,
//! and connection strings with embedded credentials. General PII (emails,
//! phone numbers, company names) is intentionally NOT blocked.
//!
//! Detection is layered, cheap-first:
//! 1. **Known-prefix / known-shape patterns** — AWS AKIA/ASIA, GitHub tokens,
//!    OpenAI `sk-proj-`, Anthropic `sk-ant-`, Stripe live keys, Fly.io tokens,
//!    Vercel secrets, Slack `xox*`, JWT triples, PEM private-key headers, Age
//!    secret keys, URL userinfo (`scheme://user:pass@`).
//! 2. **High-entropy token heuristic** — base64/hex/base64url runs ≥ 24 chars
//!    near a trigger word (key, secret, password, credential, bearer, auth,
//!    apikey, api_key, access_key, private_key). A standalone `token` still
//!    triggers opaque entropy detection, but does not by itself label a UUID
//!    as a credential; compound identifiers such as `tokenizer_*` and
//!    `token_count` remain excluded.
//!
//! Credential-shaped labels and assignments dominate the allowlist below.
//! Public VCS revisions and plausible file paths remain exempt in ordinary
//! technical prose, while path segments are still scanned independently.
//!
//! Full exemption rules (hex/UUID/SRI-hash passes, non-ASCII token
//! delimiting, structured-identifier decomposition, trigger word-boundary
//! matching, the underscore-boundary asymmetry between bare trigger words and
//! the word `token`, and the adversarial-corpus rationale for why some
//! false positives are accepted) are documented in full in
//! `docs/api/secret_gate.md#module-level-detection-algorithm` — read that before
//! changing any detection or exemption logic in this file.
//!
//! The caller-visible block message (`SecretMatch`'s `Display` impl) also
//! carries actionable guidance (`block_guidance`) to split or reword the
//! flagged token.
//!
//! A production-corpus replay harness (`corpus_replay`, `#[ignore]`d, run via
//! `KHIVE_REPLAY_DB=<path> cargo test ... -- --ignored --nocapture`) measures
//! the detector's block rate against real note/entity content; see the
//! harness's own output for current numbers rather than a point-in-time count
//! here, which would drift as the corpus changes. A checked-in, sanitized
//! snapshot of that replay (per-detector block counts and sha256 digests of
//! blocked content, never the content itself) lives at
//! `tests/data/secret_gate_corpus_manifest.md`, generated by
//! `corpus_replay::generate_corpus_manifest`.
//! A replay that is identical before and after a change certifies preservation
//! on the corpus population only, never on the shape class the change opens;
//! that class needs its own before/after arms.

use crate::error::{RuntimeError, RuntimeResult};

mod bridge_fragments;
mod credential_triggers;
mod entropy;
mod false_positive_filters;
mod known_patterns;
mod masking;
mod submitted_atom_digest;

use bridge_fragments::{
    bridge_fragment_chain, contains_bounded_word, contains_word, is_bridge_fragment_shape,
    trailing_bridge_fragment_cut,
};
use credential_triggers::{
    assignment_credential_trigger, build_match, extract_token, find_trigger,
    inline_credential_trigger, is_assignment_label_gap, is_base64_content_hash,
    is_credential_token_label, is_lookup_key_label, is_pure_hex, is_structured_identifier,
    is_uuid_canonical, shannon_entropy, strip_delimiters, value_candidates, wrapper_strip_repeated,
};
#[cfg(test)]
use credential_triggers::{LOOKUP_KEY_LABELS, POSITION_TOKEN_LABELS, RECEIPT_TOKEN_LABELS};
#[cfg(test)]
use entropy::TRIGGER_WINDOW;
use entropy::{
    check_entropy_heuristic, tokenize_entropy_tokens, EntropyScanContext, COMPOUND_TRIGGER_WORDS,
    ENTROPY_THRESHOLD, HEX_CREDENTIAL_LENGTHS, MAX_BRIDGE_FRAGMENTS, MAX_BRIDGE_GLUE_TOKENS,
    MIN_BRIDGE_FRAGMENT_LEN, MIN_ENTROPY_LEN, TRIGGER_WORDS,
};
use false_positive_filters::{
    after_last_sentence_boundary, before_first_sentence_boundary,
    has_clause_credential_label_with_inline, has_direct_repository_credential_label_with_inline,
    has_immediate_credential_label, is_aws_resource_name, is_environment_name,
    is_git_revision_reference, is_labeled_sha256_digest, is_latex_fragment_without_credential_run,
    is_latex_prose_macro, is_plausible_file_path, is_prose_code_reference,
    is_repository_revision_reference, is_vcs_marker_before_hex, normalized_hex_credential_span,
    ClauseValueKind,
};
#[cfg(test)]
use false_positive_filters::{is_clause_narrative_gerund, is_clause_narrative_participle};
use known_patterns::check_known_patterns;
#[cfg(doc)]
use known_patterns::find_url_userinfo;
#[cfg(test)]
use known_patterns::{find_prefix_token, is_filename_shaped_prefix_match, PREFIX_DETECTORS};
pub use masking::{
    bounded_masked_log_text, mask_bounded, mask_secrets, BoundedMask, MASK_WINDOW_CHARS,
};
#[cfg(test)]
use masking::{collect_mask_spans, MAX_LOG_TEXT_OUTPUT_CHARS, TRUNCATION_MARKER};
use masking::{extend_across_invisible_bridge, MAX_LOG_TEXT_MASK_INPUT_CHARS};

pub use submitted_atom_digest::masked_submitted_atom_digest_v1;

// ─── Public API ──────────────────────────────────────────────────────────────

/// Returned when a write would store credential-looking content.
///
/// Carries the detector name and a masked excerpt (`first6...Nchars`).  The
/// full candidate is never stored in the error.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SecretMatch {
    /// Human-readable name of the detector that fired.
    pub detector: &'static str,
    /// Canonical trigger from the matched context; known-prefix rules need none.
    pub trigger: Option<&'static str>,
    /// `first6...N` — the first 6 chars of the match followed by the total length.
    pub masked: String,
    /// Which record and field the match came from. `None` only where the
    /// caller scanned exactly one string and nothing else, so there is one
    /// candidate. A single-record verb that scans several fields still needs
    /// this: the writer sees one refusal and cannot tell whether it was the
    /// name, the content, a tag or a property that matched.
    pub location: Option<String>,
}

impl std::fmt::Display for SecretMatch {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "content matches secret pattern {}", self.detector)?;
        if let Some(trigger) = self.trigger {
            write!(f, " near '{trigger}'")?;
        }
        if let Some(location) = &self.location {
            write!(f, " in {location}")?;
        }
        write!(f, ". {}", block_guidance(self.detector))
    }
}

/// Actionable, caller-visible guidance for a hard block, keyed by detector
/// name. If the content genuinely is a credential, remove it. If it
/// is not — the common case for the detectors below, which key off SHAPE
/// near a trigger word rather than a known credential prefix — the fix is to
/// break up the flagged token so it no longer reads as one contiguous
/// high-entropy value: separate it from words like key/secret/auth/token with
/// a sentence or paragraph boundary, or use an explicit repository revision
/// reference for a source hash.
fn block_guidance(detector: &'static str) -> &'static str {
    match detector {
        "url-userinfo" => {
            "Placeholders in URL credential positions still match this pattern. \
             Replace the whole URL with an environment-variable name or config key, \
             or remove the entire user/password segment before writing."
        }
        "high-entropy-token"
        | "uuid-near-trigger"
        | "content-hash-near-trigger"
        | "hex-credential-token" => {
            "If this is a real credential, remove it before writing. If it is not \
             (e.g. a file path, UUID, or hash that happens to sit near a word like \
             key/secret/auth/token), put the candidate in a separate sentence or \
             paragraph from those words, or express a source hash as an explicit \
             commit/revision reference."
        }
        _ => {
            "If this is a real credential, remove it before writing; store secrets \
              in an env var or secrets manager instead."
        }
    }
}

/// Hard-block content from being written.
///
/// Returns `Err(RuntimeError::SecretDetected)` on the first match found, or
/// `Ok(())` if no secret pattern fires.
pub fn check(content: &str) -> RuntimeResult<()> {
    if let Some(m) = scan(content) {
        return Err(RuntimeError::SecretDetected(m));
    }
    Ok(())
}

/// Recursively scan a JSON value for credential-shaped strings.
///
/// Walks every string leaf (object values, array elements, nested objects).
/// Returns `Err(RuntimeError::SecretDetected)` on the first match found.
/// `None` / null / numeric / boolean JSON values are skipped.
pub fn check_json(value: &serde_json::Value) -> RuntimeResult<()> {
    scan_json_value(value)
}

/// Scan a string-tagged slice (entity/note tags).
///
/// Each tag string is scanned individually.
pub fn check_tags(tags: &[String]) -> RuntimeResult<()> {
    for tag in tags {
        check(tag)?;
    }
    Ok(())
}

/// Name the scope and field a refusal came from.
///
/// A batch verb scans each record and returns on the first refusal, so the
/// caller gets ONE error for N records. Without this the error names only the
/// matched text, which by construction is text the caller cannot find: it sits
/// in whichever sibling record refused, and every other record in the call is
/// rejected with it (khive #2605). Pass through anything that is not a gate
/// refusal unchanged — this adds identity, it does not reclassify.
///
/// `record` names the record the field belongs to, as the caller submitted it:
/// `entity`, `note`, `task`, `proposal`, `message`, indexed when it came from a
/// batch (`note[2]`). It answers where in the submitted payload the writer
/// should look, which is the question a refused writer actually asks.
pub fn locate<T>(result: RuntimeResult<T>, record: &str, field: &str) -> RuntimeResult<T> {
    result.map_err(|error| match error {
        RuntimeError::SecretDetected(matched) => RuntimeError::SecretDetected(SecretMatch {
            location: Some(format!("{record}.{field}")),
            ..matched
        }),
        other => other,
    })
}

/// `check` that names where it looked. `record` is the record noun the caller
/// submitted, indexed inside a batch; `field` is the field of it that was scanned.
pub fn check_at(content: &str, record: &str, field: &str) -> RuntimeResult<()> {
    locate(check(content), record, field)
}

/// `check_json` that names where it looked. The location is the field holding
/// the JSON, not the path of the string leaf that matched inside it.
pub fn check_json_at(value: &serde_json::Value, record: &str, field: &str) -> RuntimeResult<()> {
    locate(check_json(value), record, field)
}

/// Scan the supplied note fields in content, name, then properties order.
///
/// Returns the first refusal unchanged, located under `record` (for example,
/// `note[2].properties`). Optional fields are scanned only when present; JSON
/// scanning uses the same structured-value rules as [`check_json_at`].
///
/// This is field scanning only. Callers retain responsibility for property
/// derivation, reserved-key checks, embedding inputs, and write admission, in
/// their existing order.
pub fn check_note_fields_at(
    content: &str,
    name: Option<&str>,
    properties: Option<&serde_json::Value>,
    record: &str,
) -> RuntimeResult<()> {
    check_at(content, record, "content")?;
    if let Some(name) = name {
        check_at(name, record, "name")?;
    }
    if let Some(properties) = properties {
        check_json_at(properties, record, "properties")?;
    }
    Ok(())
}

/// `check_tags` that names where it looked.
pub fn check_tags_at(tags: &[String], record: &str, field: &str) -> RuntimeResult<()> {
    locate(check_tags(tags), record, field)
}

// ─── Reserved property keys ──────────────────────────────────────────────────

/// Top-level JSON property key reserved for runtime-owned exemption state.
///
/// No caller may create, replace, merge, or remove this key through any
/// properties-bearing write path — ADR-115 Amendment 1 §3. Reservation binds
/// unconditionally: the runtime does not yet stamp any record with this key
/// (the finalizer that would do so is a separate, later increment), so no
/// caller-supplied occurrence of it can ever be a legitimate echo of
/// persisted state. Only the exact top-level key is reserved; the same
/// spelling nested inside an object *value* is ordinary content and remains
/// subject to [`check_json`], never a posture mutation.
pub const RESERVED_SECRET_GATE_KEY: &str = "khive:secret_gate";
pub const RESERVED_WEB_RECEIPT_KEY: &str = "khive:web_receipt";
pub const WEB_RECEIPT_PROVENANCE_VALUE: &str = "v1";

/// Reject a caller-supplied top-level `khive:secret_gate` property key.
///
/// Call this before any diff, merge, or storage preparation touches
/// caller-supplied `properties` on any properties-bearing write path —
/// create, patch update, or full replace. Returns `Ok(())` when `properties`
/// is absent, is not a JSON object, or does not name the reserved key at the
/// top level.
///
/// This is the shared validator for both the ADR-115 reservation and web
/// receipt provenance. Every properties-bearing generic write path calls it
/// before a merge or full-row replacement.
pub fn reject_reserved_secret_gate_property(
    properties: Option<&serde_json::Value>,
) -> RuntimeResult<()> {
    if let Some(serde_json::Value::Object(map)) = properties {
        if map.contains_key(RESERVED_SECRET_GATE_KEY) {
            return Err(RuntimeError::InvalidInput(format!(
                "property key `{RESERVED_SECRET_GATE_KEY}` is runtime-owned and cannot be \
                 created, replaced, merged, or removed by callers"
            )));
        }
        if map.contains_key(RESERVED_WEB_RECEIPT_KEY) {
            return Err(RuntimeError::InvalidInput(format!(
                "property key `{RESERVED_WEB_RECEIPT_KEY}` is web-pack-owned and cannot be \
                 created, replaced, merged, or removed by callers"
            )));
        }
    }
    Ok(())
}

fn scan_json_value(value: &serde_json::Value) -> RuntimeResult<()> {
    match value {
        serde_json::Value::String(s) => check(s),
        serde_json::Value::Array(arr) => {
            for v in arr {
                scan_json_value(v)?;
            }
            Ok(())
        }
        serde_json::Value::Object(map) => {
            for (k, v) in map {
                // Scan both the key (a credential can appear as a JSON key name)
                // and the value recursively.
                check(k)?;
                scan_json_value(v)?;
            }
            Ok(())
        }
        _ => Ok(()),
    }
}

// ─── Scanner ─────────────────────────────────────────────────────────────────

/// Marker substituted for a detected secret span by [`mask_secrets`].
const REDACTION_MARKER: &str = "***MASKED***";

/// Maximum cumulative bytes revisited by the per-pass detector sweeps while masking
/// one input. Entropy tokens are materialized once, but resuming inside a token
/// rebuilds its member candidates and revisits their context, so that token's
/// full prefix is also charged. This permits two full-size passes over the 1 MiB
/// ASCII log-input case; the first pass is always allowed for larger or multibyte
/// callers. Once exhausted, the remainder is redacted wholesale.
const MAX_MASK_SCAN_WORK_BYTES: usize = MAX_LOG_TEXT_MASK_INPUT_CHARS * 2;

#[cfg(test)]
thread_local! {
    static ENTROPY_TOKENIZATION_COUNT: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}

/// Return the LEFTMOST secret in `text` as `(matched_slice, detector)`.
///
/// The matched slice borrows from `text`, so the caller can recover its byte
/// span via pointer arithmetic — this is what lets [`mask_secrets`] redact in
/// place while [`scan`] only needs the masked excerpt.
///
/// "Leftmost" (smallest start offset), NOT first-by-detector-priority, is the
/// load-bearing contract: [`mask_secrets`] copies the text *before* each match
/// verbatim, so a non-leftmost match would leak an earlier secret detected by a
/// lower-priority detector (e.g. an `sk-ant-` key sitting to the left of a
/// `ghp_` token). Both detector layers are folded through [`keep_leftmost`].
#[cfg(test)]
fn scan_match(text: &str) -> Option<(&str, &'static str)> {
    let context = EntropyScanContext::new(text);
    scan_from(text, 0, &context)
}

/// Like [`scan_match`], but only returns secrets whose span starts at or after
/// `from`, while still evaluating Layer-2 trigger context against the FULL
/// `text`. [`mask_secrets`] calls this with an advancing `from` so that an
/// entropy token is detected even when its only trigger word sits to the left of
/// an already-redacted earlier secret. Layer-1 known patterns are context-free,
/// so scanning the `&text[from..]` suffix is equivalent; offsets recovered via
/// pointer arithmetic against the original `text` base stay absolute. The
/// pre-tokenized entropy view is shared by all passes.
fn scan_from<'a>(
    text: &'a str,
    from: usize,
    context: &EntropyScanContext<'a>,
) -> Option<(&'a str, &'static str)> {
    scan_from_with_trigger(text, from, context).map(|(slice, detector, _)| (slice, detector))
}

fn scan_from_with_trigger<'a>(
    text: &'a str,
    from: usize,
    context: &EntropyScanContext<'a>,
) -> Option<(&'a str, &'static str, Option<&'static str>)> {
    let mut best =
        check_known_patterns(&text[from..]).map(|(slice, detector)| (slice, detector, None));
    if let Some(candidate) = check_entropy_heuristic(text, from, context) {
        if best
            .as_ref()
            .is_none_or(|current| candidate.0.as_ptr() < current.0.as_ptr())
        {
            best = Some(candidate);
        }
    }
    best
}

/// Replace `best` with `cand` when `cand` starts earlier in the original text
/// (`base` is the start address of that text). On a tie the incumbent wins, so
/// callers offer more-specific detectors first. This is what makes
/// [`check_known_patterns`] and [`scan_match`] return the leftmost secret span
/// rather than the first detector that happens to match anywhere.
fn keep_leftmost<'a>(
    best: &mut Option<(&'a str, &'static str)>,
    cand: Option<(&'a str, &'static str)>,
    base: usize,
) {
    if let Some((slice, name)) = cand {
        let start = slice.as_ptr() as usize - base;
        let replace = match *best {
            Some((incumbent, _)) => start < (incumbent.as_ptr() as usize - base),
            None => true,
        };
        if replace {
            *best = Some((slice, name));
        }
    }
}

/// Return the first `SecretMatch` found in `text`, or `None`.
fn scan(text: &str) -> Option<SecretMatch> {
    let context = EntropyScanContext::new(text);
    scan_from_with_trigger(text, 0, &context).map(|(slice, detector, trigger)| {
        let mut matched = build_match(detector, slice);
        matched.trigger = trigger;
        matched
    })
}

/// A named redact-not-block surface whose contract is intentionally separate
/// from manifest-backed write admission (ADR-115 Amendment 2).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RedactionSurface {
    /// Git ingestion stores only masked commit, issue, and pull-request text.
    GitIngest,
    /// Session mirroring stores only masked `text` and `raw` projections.
    SessionMirror,
    /// MCP diagnostics are bounded caller-visible transport data, not records.
    McpDiagnostic,
    /// The kg `scan` verb's masked preview: caller-visible, never stored.
    GateProbe,
}

/// Whether a redaction surface can consume a secret-gate exemption.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RedactionSurfaceMode {
    /// Always apply the canonical masker; never synthesize a stamp or event.
    PermanentMaskOnly,
}

/// Machine-readable contract for a named redaction surface.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RedactionSurfaceContract {
    pub mode: RedactionSurfaceMode,
    /// Durable target containing the masked result, if this is a write surface.
    pub final_stored_target: Option<&'static str>,
    /// Reserved stamp location; absent for permanent mask-only surfaces.
    pub stamp_property: Option<&'static str>,
    /// Atomic exemption-success event; absent when admission cannot occur.
    pub atomic_success_event: Option<&'static str>,
}

/// Final stored target for [`RedactionSurface::GitIngest`] — see
/// [`redaction_surface_contract`].
pub const GIT_INGEST_STORED_TARGET: &str = "final git-ingest entity/note fields";

/// Final stored target for [`RedactionSurface::SessionMirror`] — see
/// [`redaction_surface_contract`]. Names every column the session mirror
/// writes a masked provider-export projection into, not just the message
/// body columns. `cwd`/`git_branch` live only on `sessions`, keyed per
/// session — `session_messages` carries no such columns of its own (see the
/// `session_messages` DDL in `khive-pack-session`).
pub const SESSION_MIRROR_STORED_TARGET: &str =
    "session_messages.text, session_messages.raw, sessions.cwd, sessions.git_branch, \
     and sessions.slug";

/// Return the closed contract for a named redact-not-block surface.
pub const fn redaction_surface_contract(surface: RedactionSurface) -> RedactionSurfaceContract {
    let final_stored_target = match surface {
        RedactionSurface::GitIngest => Some(GIT_INGEST_STORED_TARGET),
        RedactionSurface::SessionMirror => Some(SESSION_MIRROR_STORED_TARGET),
        RedactionSurface::McpDiagnostic | RedactionSurface::GateProbe => None,
    };

    RedactionSurfaceContract {
        mode: RedactionSurfaceMode::PermanentMaskOnly,
        final_stored_target,
        stamp_property: None,
        atomic_success_event: None,
    }
}

/// Apply the canonical detector to a named permanent mask-only surface.
///
/// This wrapper makes the non-admission decision executable at each call site:
/// it has no manifest input, cannot return an exemption outcome, and cannot
/// synthesize the runtime-owned `khive:secret_gate` property or success event.
pub fn mask_for_redaction_surface(
    surface: RedactionSurface,
    text: &str,
) -> std::borrow::Cow<'_, str> {
    match redaction_surface_contract(surface).mode {
        RedactionSurfaceMode::PermanentMaskOnly => mask_secrets(text),
    }
}

#[cfg(test)]
#[path = "secret_gate/issue_2655_tests.rs"]
mod issue_2655_tests;

#[cfg(test)]
#[path = "secret_gate/note_fields_tests.rs"]
mod note_fields_tests;

// ─── Tests ───────────────────────────────────────────────────────────────────

#[cfg(test)]
#[path = "secret_gate_tests.rs"]
mod tests;

// ─── Corpus replay harness (manual, opt-in) ─────────────────────────────────
//
// Measures how many real note/entity strings the gate blocks, so a detector
// change can be evaluated against production content rather than intuition
// (see the module doc). Opens the target database
// STRICTLY read-only (`SQLITE_OPEN_READ_ONLY`) and never mutates it. Point
// `KHIVE_REPLAY_DB` at a copy or a live KG database file path; the harness
// never writes, locks aggressively, or deletes anything.
//
// Run with: `KHIVE_REPLAY_DB=/path/to/khive.db cargo test -p khive-runtime \
//   --release -- --ignored --nocapture corpus_replay`
#[cfg(test)]
#[path = "secret_gate/corpus_replay_tests.rs"]
mod corpus_replay;