khive-runtime 0.21.0

Composable Service API: entity/note CRUD, graph traversal, hybrid search, curation.
Documentation
//! Write-time secret detection gate.
//!
//! Scans caller-supplied content strings before any storage write. A match
//! causes a hard `RuntimeError::SecretDetected` that names the detector and
//! carries a masked excerpt internally. Its display names the rule and trigger
//! without echoing any candidate text.
//!
//! Scope: **credentials only** — API keys, tokens, private keys, passwords,
//! and connection strings with embedded credentials. General PII (emails,
//! phone numbers, company names) is intentionally NOT blocked.
//!
//! Detection is layered, cheap-first:
//! 1. **Known-prefix / known-shape patterns** — AWS AKIA/ASIA, GitHub tokens,
//!    OpenAI `sk-proj-`, Anthropic `sk-ant-`, Stripe live keys, Fly.io tokens,
//!    Vercel secrets, Slack `xox*`, JWT triples, PEM private-key headers, Age
//!    secret keys, URL userinfo (`scheme://user:pass@`).
//! 2. **High-entropy token heuristic** — base64/hex/base64url runs ≥ 24 chars
//!    near a trigger word (key, secret, password, credential, bearer, auth,
//!    apikey, api_key, access_key, private_key). A standalone `token` still
//!    triggers opaque entropy detection, but does not by itself label a UUID
//!    as a credential; compound identifiers such as `tokenizer_*` and
//!    `token_count` remain excluded.
//!
//! Credential-shaped labels and assignments dominate the allowlist below.
//! Public VCS revisions and plausible file paths remain exempt in ordinary
//! technical prose, while path segments are still scanned independently.
//!
//! Full exemption rules (hex/UUID/SRI-hash passes, non-ASCII token
//! delimiting, structured-identifier decomposition, trigger word-boundary
//! matching, the underscore-boundary asymmetry between bare trigger words and
//! the word `token`, and the adversarial-corpus rationale for why some
//! false positives are accepted) are documented in full in
//! `docs/api/secret_gate.md#module-level-detection-algorithm` — read that before
//! changing any detection or exemption logic in this file.
//!
//! The caller-visible block message (`SecretMatch`'s `Display` impl) also
//! carries actionable guidance (`block_guidance`) to split or reword the
//! flagged token.
//!
//! A production-corpus replay harness (`corpus_replay`, `#[ignore]`d, run via
//! `KHIVE_REPLAY_DB=<path> cargo test ... -- --ignored --nocapture`) measures
//! the detector's block rate against real note/entity content; see the
//! harness's own output for current numbers rather than a point-in-time count
//! here, which would drift as the corpus changes. A checked-in, sanitized
//! snapshot of that replay (per-detector block counts and sha256 digests of
//! blocked content, never the content itself) lives at
//! `tests/data/secret_gate_corpus_manifest.md`, generated by
//! `corpus_replay::generate_corpus_manifest`.
//! A replay that is identical before and after a change certifies preservation
//! on the corpus population only, never on the shape class the change opens;
//! that class needs its own before/after arms.

use crate::error::{RuntimeError, RuntimeResult};

#[cfg(test)]
use khive_storage::secret_scan::scan;
pub use khive_storage::secret_scan::{
    bounded_masked_log_text, mask_bounded, mask_for_redaction_surface, mask_secrets,
    masked_submitted_atom_digest_v1, redaction_surface_contract, BoundedMask, RedactionSurface,
    RedactionSurfaceContract, RedactionSurfaceMode, SecretMatch, GIT_INGEST_STORED_TARGET,
    MASK_WINDOW_CHARS, SESSION_MIRROR_STORED_TARGET, WORKSPACE_MIRROR_STORED_TARGET,
};
pub(crate) mod property_context;

/// Hard-block content from being written.
///
/// Returns `Err(RuntimeError::SecretDetected)` on the first match found, or
/// `Ok(())` if no secret pattern fires.
pub fn check(content: &str) -> RuntimeResult<()> {
    khive_storage::secret_scan::check(content).map_err(RuntimeError::SecretDetected)
}

/// Recursively scan a JSON value for credential-shaped strings.
///
/// Walks every string leaf (object values, array elements, nested objects).
/// Returns `Err(RuntimeError::SecretDetected)` on the first match found.
/// `None` / null / numeric / boolean JSON values are skipped.
pub fn check_json(value: &serde_json::Value) -> RuntimeResult<()> {
    property_context::check(value)
}

/// Scan a string-tagged slice (entity/note tags).
///
/// Each tag string is scanned individually.
pub fn check_tags(tags: &[String]) -> RuntimeResult<()> {
    for tag in tags {
        check(tag)?;
    }
    Ok(())
}

/// Name the scope and field a refusal came from.
///
/// A batch verb scans each record and returns on the first refusal, so the
/// caller gets ONE error for N records. Without this the error names only the
/// matched text, which by construction is text the caller cannot find: it sits
/// in whichever sibling record refused, and every other record in the call is
/// rejected with it (khive #2605). Pass through anything that is not a gate
/// refusal unchanged — this adds identity, it does not reclassify.
///
/// `record` names the record the field belongs to, as the caller submitted it:
/// `entity`, `note`, `task`, `proposal`, `message`, indexed when it came from a
/// batch (`note[2]`). It answers where in the submitted payload the writer
/// should look, which is the question a refused writer actually asks.
pub fn locate<T>(result: RuntimeResult<T>, record: &str, field: &str) -> RuntimeResult<T> {
    result.map_err(|error| match error {
        RuntimeError::SecretDetected(matched) => RuntimeError::SecretDetected(SecretMatch {
            location: Some(format!("{record}.{field}")),
            ..matched
        }),
        other => other,
    })
}

/// `check` that names where it looked. `record` is the record noun the caller
/// submitted, indexed inside a batch; `field` is the field of it that was scanned.
pub fn check_at(content: &str, record: &str, field: &str) -> RuntimeResult<()> {
    locate(check(content), record, field)
}

/// `check_json` that names where it looked. The location is the field holding
/// the JSON, not the path of the string leaf that matched inside it.
pub fn check_json_at(value: &serde_json::Value, record: &str, field: &str) -> RuntimeResult<()> {
    locate(check_json(value), record, field)
}

/// Scan the supplied note fields in content, name, then properties order.
///
/// Returns the first refusal unchanged, located under `record` (for example,
/// `note[2].properties`). Optional fields are scanned only when present; JSON
/// scanning uses the same structured-value rules as [`check_json_at`].
///
/// This is field scanning only. Callers retain responsibility for property
/// derivation, reserved-key checks, embedding inputs, and write admission, in
/// their existing order.
pub fn check_note_fields_at(
    content: &str,
    name: Option<&str>,
    properties: Option<&serde_json::Value>,
    record: &str,
) -> RuntimeResult<()> {
    check_at(content, record, "content")?;
    if let Some(name) = name {
        check_at(name, record, "name")?;
    }
    if let Some(properties) = properties {
        check_json_at(properties, record, "properties")?;
    }
    Ok(())
}

/// `check_tags` that names where it looked.
pub fn check_tags_at(tags: &[String], record: &str, field: &str) -> RuntimeResult<()> {
    locate(check_tags(tags), record, field)
}

// ─── Reserved property keys ──────────────────────────────────────────────────

/// Top-level JSON property key reserved for runtime-owned exemption state.
///
/// No caller may create, replace, merge, or remove this key through any
/// properties-bearing write path — ADR-115 Amendment 1 §3. Reservation binds
/// unconditionally: the runtime does not yet stamp any record with this key
/// (the finalizer that would do so is a separate, later increment), so no
/// caller-supplied occurrence of it can ever be a legitimate echo of
/// persisted state. Only the exact top-level key is reserved; the same
/// spelling nested inside an object *value* is ordinary content and remains
/// subject to [`check_json`], never a posture mutation.
pub const RESERVED_SECRET_GATE_KEY: &str = "khive:secret_gate";
pub const RESERVED_WEB_RECEIPT_KEY: &str = "khive:web_receipt";
pub const WEB_RECEIPT_PROVENANCE_VALUE: &str = "v1";

/// Reject a caller-supplied top-level `khive:secret_gate` property key.
///
/// Call this before any diff, merge, or storage preparation touches
/// caller-supplied `properties` on any properties-bearing write path —
/// create, patch update, or full replace. Returns `Ok(())` when `properties`
/// is absent, is not a JSON object, or does not name the reserved key at the
/// top level.
///
/// This is the shared validator for both the ADR-115 reservation and web
/// receipt provenance. Every properties-bearing generic write path calls it
/// before a merge or full-row replacement.
pub fn reject_reserved_secret_gate_property(
    properties: Option<&serde_json::Value>,
) -> RuntimeResult<()> {
    if let Some(serde_json::Value::Object(map)) = properties {
        if let Some(key) = RESERVED_PROPERTY_KEYS
            .into_iter()
            .find(|key| map.contains_key(*key))
        {
            return Err(reserved_property_error(key));
        }
    }
    Ok(())
}

/// The top-level property keys no caller may set, in the order their refusals are reported.
pub(crate) const RESERVED_PROPERTY_KEYS: [&str; 2] =
    [RESERVED_SECRET_GATE_KEY, RESERVED_WEB_RECEIPT_KEY];

fn reserved_property_error(key: &str) -> RuntimeError {
    let owner = if key == RESERVED_WEB_RECEIPT_KEY {
        "web-pack-owned"
    } else {
        "runtime-owned"
    };
    RuntimeError::InvalidInput(format!(
        "property key `{key}` is {owner} and cannot be created, replaced, merged, or removed by \
         callers"
    ))
}

/// ADR-115 §4 for a whole-map property write against a persisted row.
///
/// An incoming reserved key whose value is byte-identical to the persisted value is an echo
/// of runtime state and is removed from `incoming`. A reserved key the row does not carry, or
/// one whose value differs (`null` included), is refused. Omitting a reserved key from a
/// whole map is not a removal: whether the persisted value carries forward is the caller's
/// decision, conditioned on the exempted bytes (see [`reserved_properties`]).
pub(crate) fn strip_reserved_echo(
    incoming: &mut serde_json::Value,
    persisted: Option<&serde_json::Value>,
) -> RuntimeResult<()> {
    let Some(map) = incoming.as_object_mut() else {
        return Ok(());
    };
    let persisted = persisted.and_then(serde_json::Value::as_object);
    for key in RESERVED_PROPERTY_KEYS {
        let Some(value) = map.get(key) else {
            continue;
        };
        let echo = persisted
            .and_then(|row| row.get(key))
            .is_some_and(|stored| same_bytes(stored, value));
        if !echo {
            return Err(reserved_property_error(key));
        }
        map.remove(key);
    }
    Ok(())
}

fn same_bytes(left: &serde_json::Value, right: &serde_json::Value) -> bool {
    matches!(
        (serde_json::to_vec(left), serde_json::to_vec(right)),
        (Ok(left), Ok(right)) if left == right
    )
}

/// The reserved entries a persisted property map carries, in [`RESERVED_PROPERTY_KEYS`] order.
pub(crate) fn reserved_properties(
    properties: Option<&serde_json::Value>,
) -> Vec<(&'static str, serde_json::Value)> {
    let Some(map) = properties.and_then(serde_json::Value::as_object) else {
        return Vec::new();
    };
    RESERVED_PROPERTY_KEYS
        .into_iter()
        .filter_map(|key| map.get(key).map(|value| (key, value.clone())))
        .collect()
}

/// `properties` with the reserved keys removed: the part of a property map the secret gate
/// scans, and so the part an exemption stamp vouches for.
pub(crate) fn without_reserved_properties(
    properties: Option<&serde_json::Value>,
) -> Option<serde_json::Value> {
    let mut properties = properties.cloned();
    if let Some(serde_json::Value::Object(map)) = properties.as_mut() {
        for key in RESERVED_PROPERTY_KEYS {
            map.remove(key);
        }
    }
    properties
}

/// The write-site check for ADR-115 §4 on a whole-row replace of a row that may carry a
/// reserved key. A reserved value may reach the row only as a byte-identical carry of the
/// value `persisted` already holds, and only while the bytes it vouches for are unchanged
/// (`vouched_unchanged`). Every such write site calls this immediately before it builds the
/// write, whatever its preparation checked, so a preparation that drifts cannot persist a
/// stamp over bytes it never covered.
pub(crate) fn check_reserved_carry_forward(
    persisted: Option<&serde_json::Value>,
    outgoing: Option<&serde_json::Value>,
    vouched_unchanged: bool,
) -> RuntimeResult<()> {
    let persisted = reserved_properties(persisted);
    for (key, value) in reserved_properties(outgoing) {
        let carried = persisted
            .iter()
            .any(|(stored_key, stored)| *stored_key == key && same_bytes(stored, &value));
        if !carried || !vouched_unchanged {
            return Err(reserved_property_error(key));
        }
    }
    Ok(())
}

#[cfg(test)]
#[path = "secret_gate/note_fields_tests.rs"]
mod note_fields_tests;

#[cfg(test)]
#[path = "secret_gate/escaped_json_tests.rs"]
mod escaped_json_tests;

// ─── Tests ───────────────────────────────────────────────────────────────────

#[cfg(test)]
#[path = "secret_gate_tests.rs"]
mod tests;

// ─── Corpus replay harness (manual, opt-in) ─────────────────────────────────
//
// Measures how many real note/entity strings the gate blocks, so a detector
// change can be evaluated against production content rather than intuition
// (see the module doc). Opens the target database
// STRICTLY read-only (`SQLITE_OPEN_READ_ONLY`) and never mutates it. Point
// `KHIVE_REPLAY_DB` at a copy or a live KG database file path; the harness
// never writes, locks aggressively, or deletes anything.
//
// Run with: `KHIVE_REPLAY_DB=/path/to/khive.db cargo test -p khive-runtime \
//   --release -- --ignored --nocapture corpus_replay`
#[cfg(test)]
#[path = "secret_gate/corpus_replay_tests.rs"]
mod corpus_replay;