lean_ctx/core/policy/

coverage.rs

//! CGB coverage — automated *partial* assessment of a resolved policy pack
//! against the Context Governance Benchmark v1.0-draft (GL #426).
//!
//! Honesty contract: a static pack analysis can only ever produce **partial
//! evidence** for a handful of controls. This module therefore (a) checks
//! redaction patterns against real synthetic fixtures instead of trusting
//! pattern names, (b) reports `Inconclusive` — never `Pass` — when a pack
//! simply doesn't state something, and (c) refuses to compute a maturity
//! grade: grades require the full manual assessment
//! (`assessment/TEMPLATE.md` in the spec repo).

use regex::Regex;
use serde::Serialize;

use super::ResolvedPolicy;

/// Spec version the checks below were written against.
pub const BENCHMARK_ID: &str = "cgb-v1.0-draft";
/// Total controls in the spec — denominator for the honesty line.
pub const CONTROLS_TOTAL: usize = 32;

/// Outcome of one automated check.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum CheckStatus {
    /// The pack provides positive evidence for this aspect.
    Pass,
    /// The pack contradicts the control's expectation.
    Fail,
    /// The pack is silent — the aspect must be verified elsewhere.
    Inconclusive,
}

/// One automated check over one control aspect.
#[derive(Debug, Clone, Serialize)]
pub struct CoverageCheck {
    /// Control ID, e.g. `CGB-1.1`.
    pub control: &'static str,
    /// Short control title (spec wording, abbreviated).
    pub title: &'static str,
    pub status: CheckStatus,
    /// What was observed, in one line.
    pub detail: String,
}

/// Synthetic credential fixtures for CGB-1.1 — one per credential class the
/// control names. A class counts as covered when any resolved redaction
/// pattern matches its fixture.
pub(crate) const CREDENTIAL_FIXTURES: &[(&str, &str)] = &[
    ("private key block", "-----BEGIN RSA PRIVATE KEY-----"),
    ("cloud access key", concat!("AK", "IAIOSFODNN7EXAMPLE")),
    (
        "credential assignment",
        concat!("api_key = \"sk", "-supersecretvalue1234\""),
    ),
    (
        "bearer token",
        "Authorization: Bearer abcdefghij0123456789xyz",
    ),
];

/// Synthetic non-credential sensitivity fixtures for CGB-1.3 (regulated
/// identifiers). Matching ≥ 1 demonstrates classification beyond secrets.
pub(crate) const DOMAIN_FIXTURES: &[(&str, &str)] = &[
    ("IBAN", "DE89 3704 0044 0532 0130 00"),
    ("payment card", "4111 1111 1111 1111"),
    ("US SSN", "SSN: 123-45-6789"),
    ("date of birth", "DOB: 03/14/1975"),
];

/// Tools that reach the network beyond the model provider (CGB-5.5 aspect:
/// egress deniable by policy). An explicit list, not a name heuristic
/// (GHSA-jhf9-3pv4-m469): `ctx_provider` (GitHub/GitLab/Jira/Postgres) and
/// `ctx_tools` (MCP gateway) do not look like fetch tools but egress all the
/// same. `ctx_preload`'s provider prefetch is gated on `ctx_provider`.
pub(crate) const EGRESS_CAPABLE_TOOLS: &[&str] = &["ctx_url_read", "ctx_provider", "ctx_tools"];

/// Egress-capable tools a resolved pack still permits (same allow/deny
/// semantics as the runtime gate: deny wins, an allowlist is exclusive).
pub(crate) fn egress_tools_permitted(policy: &ResolvedPolicy) -> Vec<&'static str> {
    EGRESS_CAPABLE_TOOLS
        .iter()
        .copied()
        .filter(|tool| {
            !policy.deny_tools.iter().any(|t| t == tool)
                && policy
                    .allow_tools
                    .as_ref()
                    .is_none_or(|allow| allow.iter().any(|t| t == tool))
        })
        .collect()
}

/// Run all automated checks against a resolved policy.
pub fn assess(policy: &ResolvedPolicy) -> Vec<CoverageCheck> {
    let patterns: Vec<(String, Regex)> = policy
        .redaction
        .iter()
        .filter_map(|(name, raw)| Regex::new(raw).ok().map(|re| (name.clone(), re)))
        .collect();

    let matches = |fixture: &str| patterns.iter().any(|(_, re)| re.is_match(fixture));

    let mut checks = Vec::new();

    // CGB-1.1 — credential material never reaches the model.
    let missing: Vec<&str> = CREDENTIAL_FIXTURES
        .iter()
        .filter(|(_, fixture)| !matches(fixture))
        .map(|(class, _)| *class)
        .collect();
    checks.push(if missing.is_empty() {
        CoverageCheck {
            control: "CGB-1.1",
            title: "credential redaction",
            status: CheckStatus::Pass,
            detail: format!(
                "{}/{} credential fixture classes matched by redaction patterns",
                CREDENTIAL_FIXTURES.len(),
                CREDENTIAL_FIXTURES.len()
            ),
        }
    } else {
        CoverageCheck {
            control: "CGB-1.1",
            title: "credential redaction",
            status: CheckStatus::Fail,
            detail: format!("unredacted credential classes: {}", missing.join(", ")),
        }
    });

    // CGB-1.2 — declarative, reviewable rules (named patterns in TOML).
    checks.push(if policy.redaction.is_empty() {
        CoverageCheck {
            control: "CGB-1.2",
            title: "declarative redaction rules",
            status: CheckStatus::Fail,
            detail: "pack declares no named redaction patterns".to_string(),
        }
    } else {
        CoverageCheck {
            control: "CGB-1.2",
            title: "declarative redaction rules",
            status: CheckStatus::Pass,
            detail: format!(
                "{} named, versioned patterns (chain: {})",
                policy.redaction.len(),
                if policy.chain.is_empty() {
                    "root pack".to_string()
                } else {
                    policy.chain.join(" → ")
                }
            ),
        }
    });

    // CGB-1.3 — classification beyond secrets (regulated identifiers).
    let domain_hits: Vec<&str> = DOMAIN_FIXTURES
        .iter()
        .filter(|(_, fixture)| matches(fixture))
        .map(|(class, _)| *class)
        .collect();
    checks.push(if domain_hits.is_empty() {
        CoverageCheck {
            control: "CGB-1.3",
            title: "beyond-secret classification",
            status: CheckStatus::Inconclusive,
            detail:
                "no regulated-identifier patterns declared — acceptable outside regulated workloads"
                    .to_string(),
        }
    } else {
        CoverageCheck {
            control: "CGB-1.3",
            title: "beyond-secret classification",
            status: CheckStatus::Pass,
            detail: format!("regulated classes redacted: {}", domain_hits.join(", ")),
        }
    });

    // CGB-3.2 — hard budget configured (pack aspect: context budget cap).
    checks.push(match policy.max_context_tokens {
        Some(cap) => CoverageCheck {
            control: "CGB-3.2",
            title: "context budget cap",
            status: CheckStatus::Pass,
            detail: format!("max_context_tokens = {cap}"),
        },
        None => CoverageCheck {
            control: "CGB-3.2",
            title: "context budget cap",
            status: CheckStatus::Inconclusive,
            detail: "no cap in pack — verify budget enforcement elsewhere".to_string(),
        },
    });

    // CGB-4.3 — retention policy-driven (pack aspect: declared expectation).
    checks.push(match policy.audit_retention_days {
        Some(days) => CoverageCheck {
            control: "CGB-4.3",
            title: "audit retention declared",
            status: CheckStatus::Pass,
            detail: format!("audit_retention_days = {days}"),
        },
        None => CoverageCheck {
            control: "CGB-4.3",
            title: "audit retention declared",
            status: CheckStatus::Inconclusive,
            detail: "no retention expectation in pack".to_string(),
        },
    });

    // CGB-5.4 — capabilities scoped (pack aspect: tool allow/deny posture).
    let denies = policy.deny_tools.len();
    checks.push(match (&policy.allow_tools, denies) {
        (Some(allow), _) => CoverageCheck {
            control: "CGB-5.4",
            title: "tool surface scoped",
            status: CheckStatus::Pass,
            detail: format!(
                "allowlist posture: {} tools permitted, rest denied",
                allow.len()
            ),
        },
        (None, d) if d > 0 => CoverageCheck {
            control: "CGB-5.4",
            title: "tool surface scoped",
            status: CheckStatus::Pass,
            detail: format!("denylist posture: {d} denied tool(s)"),
        },
        _ => CoverageCheck {
            control: "CGB-5.4",
            title: "tool surface scoped",
            status: CheckStatus::Inconclusive,
            detail: "pack neither allows nor denies tools — instance defaults apply".to_string(),
        },
    });

    // CGB-5.5 — egress governed (pack aspect: egress tools restricted).
    // PASS only when every egress-capable tool is blocked; a pack that
    // restricts some of them but not all FAILs and names the open ones.
    let still_open = egress_tools_permitted(policy);
    checks.push(if still_open.is_empty() {
        CoverageCheck {
            control: "CGB-5.5",
            title: "egress restricted",
            status: CheckStatus::Pass,
            detail: format!(
                "all egress-capable tools blocked: {}",
                EGRESS_CAPABLE_TOOLS.join(", ")
            ),
        }
    } else if still_open.len() < EGRESS_CAPABLE_TOOLS.len() {
        CoverageCheck {
            control: "CGB-5.5",
            title: "egress restricted",
            status: CheckStatus::Fail,
            detail: format!(
                "egress only partially restricted — still permitted: {}",
                still_open.join(", ")
            ),
        }
    } else {
        CoverageCheck {
            control: "CGB-5.5",
            title: "egress restricted",
            status: CheckStatus::Inconclusive,
            detail: "pack does not restrict egress tools — verify via roles/network policy"
                .to_string(),
        }
    });

    checks
}

/// Counts per status, for the summary line and JSON.
#[derive(Debug, Serialize)]
pub struct CoverageSummary {
    pub pass: usize,
    pub fail: usize,
    pub inconclusive: usize,
    /// Distinct controls the automated checks touch.
    pub controls_covered: usize,
    pub controls_total: usize,
}

/// Summarize a check run.
pub fn summarize(checks: &[CoverageCheck]) -> CoverageSummary {
    let mut covered: Vec<&str> = checks.iter().map(|c| c.control).collect();
    covered.dedup();
    CoverageSummary {
        pass: checks
            .iter()
            .filter(|c| c.status == CheckStatus::Pass)
            .count(),
        fail: checks
            .iter()
            .filter(|c| c.status == CheckStatus::Fail)
            .count(),
        inconclusive: checks
            .iter()
            .filter(|c| c.status == CheckStatus::Inconclusive)
            .count(),
        controls_covered: covered.len(),
        controls_total: CONTROLS_TOTAL,
    }
}

#[cfg(test)]
mod tests {
    use super::super::builtin;
    use super::*;

    fn resolved(name: &str) -> ResolvedPolicy {
        let pack = builtin::get(name).expect("built-in exists");
        super::super::resolve(&pack).expect("resolves")
    }

    /// A user-authored regulated pack: own identifier patterns on top of the
    /// community floor, web egress denied, budget and retention pinned.
    fn regulated() -> ResolvedPolicy {
        let pack = super::super::parse(
            r#"
name = "regulated-test"
version = "1.0.0"
description = "test pack with own identifier classes"
extends = "strict-redaction"

[context]
deny_tools = ["ctx_url_read", "ctx_provider", "ctx_tools"]
max_context_tokens = 12000
audit_retention_days = 365

[redaction]
iban = '\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b'
us_ssn = '\b\d{3}-\d{2}-\d{4}\b'
"#,
        )
        .expect("parses");
        super::super::resolve(&pack).expect("resolves")
    }

    fn status_of(checks: &[CoverageCheck], control: &str) -> CheckStatus {
        checks
            .iter()
            .find(|c| c.control == control)
            .expect("control checked")
            .status
    }

    #[test]
    fn baseline_passes_credential_redaction() {
        let checks = assess(&resolved("baseline"));
        assert_eq!(status_of(&checks, "CGB-1.1"), CheckStatus::Pass);
        assert_eq!(status_of(&checks, "CGB-1.2"), CheckStatus::Pass);
        // baseline has no regulated-identifier classes and no tool posture.
        assert_eq!(status_of(&checks, "CGB-1.3"), CheckStatus::Inconclusive);
        assert_eq!(status_of(&checks, "CGB-5.4"), CheckStatus::Inconclusive);
    }

    #[test]
    fn regulated_pack_demonstrates_domain_classes_and_egress_denial() {
        let checks = assess(&regulated());
        assert_eq!(status_of(&checks, "CGB-1.1"), CheckStatus::Pass);
        assert_eq!(status_of(&checks, "CGB-1.3"), CheckStatus::Pass);
        assert_eq!(status_of(&checks, "CGB-3.2"), CheckStatus::Pass);
        assert_eq!(status_of(&checks, "CGB-4.3"), CheckStatus::Pass);
        assert_eq!(status_of(&checks, "CGB-5.5"), CheckStatus::Pass);
    }

    /// GHSA-jhf9-3pv4-m469: denying only `ctx_url_read` is not "egress
    /// restricted" while providers and the MCP gateway still egress.
    #[test]
    fn partial_egress_denial_fails_and_names_open_tools() {
        let mut policy = regulated();
        assert_eq!(status_of(&assess(&policy), "CGB-5.5"), CheckStatus::Pass);
        policy.deny_tools = vec!["ctx_url_read".into()];
        let checks = assess(&policy);
        let check = checks.iter().find(|c| c.control == "CGB-5.5").unwrap();
        assert_eq!(check.status, CheckStatus::Fail);
        assert!(check.detail.contains("ctx_provider"), "{}", check.detail);
        assert!(check.detail.contains("ctx_tools"), "{}", check.detail);
        policy.deny_tools.clear();
        policy.allow_tools = Some(vec!["ctx_read".into()]);
        assert_eq!(status_of(&assess(&policy), "CGB-5.5"), CheckStatus::Pass);
    }

    #[test]
    fn empty_policy_fails_credential_checks() {
        let empty = ResolvedPolicy {
            name: "empty".into(),
            version: "0.0.1".into(),
            description: String::new(),
            chain: vec![],
            default_read_mode: None,
            allow_tools: None,
            deny_tools: vec![],
            max_context_tokens: None,
            audit_retention_days: None,
            redaction: std::collections::BTreeMap::new(),
            filters: crate::core::policy::FilterRules::default(),
            egress: crate::core::policy::EgressRules::default(),
            routing: crate::core::policy::RoutingPolicyRules::default(),
            budgets: crate::core::policy::BudgetRules::default(),
        };
        let checks = assess(&empty);
        assert_eq!(status_of(&checks, "CGB-1.1"), CheckStatus::Fail);
        assert_eq!(status_of(&checks, "CGB-1.2"), CheckStatus::Fail);
    }

    #[test]
    fn summary_counts_are_consistent() {
        let checks = assess(&regulated());
        let s = summarize(&checks);
        assert_eq!(s.pass + s.fail + s.inconclusive, checks.len());
        assert_eq!(s.controls_total, CONTROLS_TOTAL);
        assert!(s.controls_covered <= checks.len());
    }
}