use assert_cmd::Command;
use std::fs;
use std::path::Path;
use std::process::Command as StdCommand;
type TestResult = Result<(), Box<dyn std::error::Error>>;
const MANIFEST: &str = r#"
version = 1
canonical = "json-schema-2020-12"
authoring = "zod"
[[contracts]]
name = "user"
source = "contracts/user.zod.ts"
emit = ["zod"]
[[mappings]]
glob = "app/api/**/*.ts"
contracts = ["user"]
require = "boundary-validation"
[gates]
suppression_comments = "deny"
protected_paths = ["pushkin.toml"]
read_only_paths = ["crates/**/tests/**"]
retrieval_paths = ["crates/**/*.rs"]
retrieval_tool = "mcp__codebase-retrieval__codebase-retrieval"
"#;
const PROTECTED: &str = "pushkin.toml";
const COMMITTED_TEST: &str = "crates/pushkin-cli/tests/committed_suite.rs";
const SOURCE: &str = "crates/pushkin-cli/src/main.rs";
const MAPPED: &str = "app/api/users/route.ts";
const UNGATED: &str = "docs/notes.md";
const RULE_UNVALIDATED: &str = "contract.boundary.unvalidated_input";
const RULE_PROTECTED: &str = "pushkin.protected_path";
const RULE_READ_ONLY: &str = "pushkin.read_only_path";
const RULE_RAW_READ: &str = "pushkin.retrieval.raw_read";
const FAIL_OPEN_MARKER: &str = "failing open";
fn git(dir: &Path, args: &[&str]) -> TestResult {
let status = StdCommand::new("git")
.current_dir(dir)
.args(args)
.status()?;
assert!(status.success(), "git {args:?} failed");
Ok(())
}
fn write_fixture_tree(dir: &Path) -> TestResult {
fs::write(dir.join(PROTECTED), MANIFEST)?;
fs::create_dir_all(dir.join("contracts"))?;
fs::write(
dir.join("contracts/user.zod.ts"),
"export const user = 1;\n",
)?;
fs::create_dir_all(dir.join("app/api/users"))?;
fs::write(dir.join(MAPPED), "export const handler = 1;\n")?;
fs::create_dir_all(dir.join("crates/pushkin-cli/tests"))?;
fs::write(dir.join(COMMITTED_TEST), "// committed suite\n")?;
fs::create_dir_all(dir.join("crates/pushkin-cli/src"))?;
fs::write(dir.join(SOURCE), "fn main() {}\n")?;
fs::create_dir_all(dir.join("docs"))?;
fs::write(dir.join(UNGATED), "notes\n")?;
Ok(())
}
fn repo() -> Result<tempfile::TempDir, Box<dyn std::error::Error>> {
let dir = tempfile::tempdir()?;
write_fixture_tree(dir.path())?;
git(dir.path(), &["init", "-q", "."])?;
git(dir.path(), &["add", "-A"])?;
git(
dir.path(),
&[
"-c",
"user.name=Conformance Suite",
"-c",
"user.email=conformance@test",
"commit",
"-qm",
"fixture",
],
)?;
Ok(dir)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Decision {
Allow,
Block,
FailOpen,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Verdict {
decision: Decision,
violations: Vec<String>,
}
struct Case {
name: &'static str,
check_payload: &'static str,
agent: &'static str,
hook_payload: &'static str,
daemon: &'static str,
expect: Expect,
}
enum Expect {
Agree,
Diverges {
check: Decision,
hook: Decision,
why: &'static str,
},
}
fn key_from_line(line: &str) -> Option<String> {
let (head, rest) = line.split_once(" [")?;
let rule = rest.split_once(']')?.0;
let head = head.trim();
let (file, line_no) = head.rsplit_once(':')?;
if file.is_empty() || line_no.is_empty() || !line_no.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
Some(format!("{file}:{line_no} [{rule}]"))
}
fn violation_keys(prose: &str) -> Vec<String> {
let mut keys: Vec<String> = prose.lines().filter_map(key_from_line).collect();
keys.sort();
keys.dedup();
keys
}
#[derive(Debug)]
struct Raw {
code: i32,
stdout: String,
stderr: String,
}
impl Raw {
fn failed_open(&self) -> bool {
self.stderr.contains(FAIL_OPEN_MARKER)
}
}
struct Invocation<'a> {
args: &'a [&'a str],
daemon: &'a str,
payload: &'a str,
}
fn run(dir: &Path, call: &Invocation) -> Result<Raw, Box<dyn std::error::Error>> {
let output = Command::cargo_bin("pushkin")?
.current_dir(dir)
.env("PUSHKIN_DAEMON", call.daemon)
.args(call.args)
.write_stdin(call.payload.to_owned())
.output()?;
Ok(Raw {
code: output.status.code().unwrap_or(-1),
stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
})
}
fn abstained(raw: &Raw, case: &str) -> Verdict {
assert!(
raw.failed_open(),
"{case}: check produced no envelope and did not fail open: {raw:?}"
);
Verdict {
decision: Decision::FailOpen,
violations: Vec::new(),
}
}
fn check_verdict(dir: &Path, case: &Case) -> Result<Verdict, Box<dyn std::error::Error>> {
let raw = run(
dir,
&Invocation {
args: &["check", "--json"],
daemon: case.daemon,
payload: case.check_payload,
},
)?;
if raw.stdout.trim().is_empty() {
return Ok(abstained(&raw, case.name));
}
let envelope: serde_json::Value = serde_json::from_str(&raw.stdout)?;
let decision = match envelope["decision"].as_str() {
Some("allow") => Decision::Allow,
Some("block") => Decision::Block,
other => {
return Err(format!("{}: unknown check decision {other:?}: {raw:?}", case.name).into())
}
};
assert_code_corroborates(&raw, decision);
Ok(Verdict {
decision,
violations: envelope_keys(&envelope),
})
}
fn assert_code_corroborates(raw: &Raw, decision: Decision) {
let expected = if decision == Decision::Block { 2 } else { 0 };
assert_eq!(
raw.code, expected,
"check's exit code contradicts its own envelope: {raw:?}"
);
}
fn envelope_keys(envelope: &serde_json::Value) -> Vec<String> {
let mut keys: Vec<String> = envelope["violations"]
.as_array()
.into_iter()
.flatten()
.map(|v| {
format!(
"{}:{} [{}]",
v["file"].as_str().unwrap_or("?"),
v["line"].as_u64().unwrap_or(0),
v["rule"].as_str().unwrap_or("?")
)
})
.collect();
keys.sort();
keys.dedup();
keys
}
fn hook_verdict(dir: &Path, case: &Case) -> Result<Verdict, Box<dyn std::error::Error>> {
let raw = run(
dir,
&Invocation {
args: &["hook", case.agent],
daemon: case.daemon,
payload: case.hook_payload,
},
)?;
let Some(prose) = deny_prose(&raw.stdout) else {
return Ok(Verdict {
decision: if raw.failed_open() {
Decision::FailOpen
} else {
Decision::Allow
},
violations: Vec::new(),
});
};
let violations = violation_keys(&prose);
assert!(
!violations.is_empty(),
"{}: hook denied but no violation line parsed — the ladder format moved: {prose}",
case.name
);
Ok(Verdict {
decision: Decision::Block,
violations,
})
}
fn deny_prose(stdout: &str) -> Option<String> {
let value: serde_json::Value = serde_json::from_str(stdout.trim()).ok()?;
for field in [
"/hookSpecificOutput/permissionDecisionReason",
"/reason",
"/message",
] {
if let Some(prose) = value.pointer(field).and_then(serde_json::Value::as_str) {
return Some(prose.to_owned());
}
}
None
}
fn assert_conformance(dir: &Path, case: &Case) -> TestResult {
let check = check_verdict(dir, case)?;
let hook = hook_verdict(dir, case)?;
match case.expect {
Expect::Agree => assert_eq!(
check, hook,
"{}: the two surfaces rendered DIFFERENT verdicts for one logical \
action. check={check:?} hook={hook:?}",
case.name
),
Expect::Diverges {
check: want_check,
hook: want_hook,
why,
} => {
assert_ne!(
check, hook,
"{}: the surfaces now AGREE, but this row is recorded as \
divergent ({why}). If the divergence was fixed, reclassify this \
row to Expect::Agree and close the finding. check={check:?}",
case.name
);
assert_eq!(
(check.decision, hook.decision),
(want_check, want_hook),
"{}: the surfaces still disagree, but not in the recorded \
SHAPE ({why}). check={check:?} hook={hook:?}",
case.name
);
}
}
Ok(())
}
const WRITE_NONCONFORMING: Case = Case {
name: "write_nonconforming_to_mapped_glob",
check_payload: r#"{"session_id":"gdc-01","tool_name":"Write","tool_input":{"file_path":"app/api/users/route.ts","content":"export async function POST(req){ const b = await req.json(); return Response.json(b); }"}}"#,
agent: "claude",
hook_payload: r#"{"session_id":"gdc-01","tool_name":"Write","tool_input":{"file_path":"app/api/users/route.ts","content":"export async function POST(req){ const b = await req.json(); return Response.json(b); }"}}"#,
daemon: "off",
expect: Expect::Agree,
};
const WRITE_NONCONFORMING_WARM: Case = Case {
name: "write_nonconforming_to_mapped_glob_warm",
daemon: "auto",
..WRITE_NONCONFORMING
};
const WRITE_CONFORMING: Case = Case {
name: "write_conforming_to_mapped_glob",
check_payload: r#"{"session_id":"gdc-02","tool_name":"Write","tool_input":{"file_path":"app/api/users/route.ts","content":"import { UserCreateSchema } from \"contracts/user.zod\";\nexport async function POST(req){ const b = UserCreateSchema.parse(await req.json()); return Response.json(b); }"}}"#,
agent: "claude",
hook_payload: r#"{"session_id":"gdc-02","tool_name":"Write","tool_input":{"file_path":"app/api/users/route.ts","content":"import { UserCreateSchema } from \"contracts/user.zod\";\nexport async function POST(req){ const b = UserCreateSchema.parse(await req.json()); return Response.json(b); }"}}"#,
daemon: "off",
expect: Expect::Agree,
};
const WRITE_UNGATED: Case = Case {
name: "write_to_ungated_unmapped_path",
check_payload: r#"{"session_id":"gdc-03","tool_name":"Write","tool_input":{"file_path":"docs/notes.md","content":"hello"}}"#,
agent: "claude",
hook_payload: r#"{"session_id":"gdc-03","tool_name":"Write","tool_input":{"file_path":"docs/notes.md","content":"hello"}}"#,
daemon: "off",
expect: Expect::Agree,
};
const WRITE_PROTECTED: Case = Case {
name: "write_to_protected_path",
check_payload: r#"{"session_id":"gdc-04","tool_name":"Write","tool_input":{"file_path":"pushkin.toml","content":"version = 1"}}"#,
agent: "claude",
hook_payload: r#"{"session_id":"gdc-04","tool_name":"Write","tool_input":{"file_path":"pushkin.toml","content":"version = 1"}}"#,
daemon: "off",
expect: Expect::Agree,
};
const EDIT_READ_ONLY: Case = Case {
name: "edit_a_committed_read_only_test",
check_payload: r#"{"session_id":"gdc-05","tool_name":"Edit","tool_input":{"file_path":"crates/pushkin-cli/tests/committed_suite.rs","old_string":"// committed suite","new_string":"// tampered"}}"#,
agent: "claude",
hook_payload: r#"{"session_id":"gdc-05","tool_name":"Edit","tool_input":{"file_path":"crates/pushkin-cli/tests/committed_suite.rs","old_string":"// committed suite","new_string":"// tampered"}}"#,
daemon: "off",
expect: Expect::Agree,
};
const MULTIEDIT_READ_ONLY: Case = Case {
name: "multiedit_a_committed_read_only_test",
check_payload: r#"{"session_id":"gdc-06","tool_name":"MultiEdit","tool_input":{"file_path":"crates/pushkin-cli/tests/committed_suite.rs","edits":[{"old_string":"// committed suite","new_string":"// tampered"}]}}"#,
agent: "claude",
hook_payload: r#"{"session_id":"gdc-06","tool_name":"MultiEdit","tool_input":{"file_path":"crates/pushkin-cli/tests/committed_suite.rs","edits":[{"old_string":"// committed suite","new_string":"// tampered"}]}}"#,
daemon: "off",
expect: Expect::Agree,
};
const STOP_SWEEP: Case = Case {
name: "stop_payload_sweeps_the_repo",
check_payload: r#"{"session_id":"gdc-07","stop_hook_active":true}"#,
agent: "claude",
hook_payload: r#"{"session_id":"gdc-07","stop_hook_active":true}"#,
daemon: "off",
expect: Expect::Agree,
};
const UNREADABLE_NAMING_PROTECTED: Case = Case {
name: "unreadable_payload_naming_a_protected_path",
check_payload: r#"{"session_id":"gdc-08","tool_name":"SomeToolPushkinHasNeverHeardOf","tool_input":{"target":"pushkin.toml"}}"#,
agent: "claude",
hook_payload: r#"{"session_id":"gdc-08","tool_name":"SomeToolPushkinHasNeverHeardOf","tool_input":{"target":"pushkin.toml"}}"#,
daemon: "off",
expect: Expect::Agree,
};
const CODEX_DELETE_READ_ONLY: Case = Case {
name: "codex_apply_patch_delete_of_a_read_only_test",
check_payload: r#"{"session_id":"gdc-09","tool_name":"apply_patch","tool_input":{"command":"*** Begin Patch\n*** Delete File: crates/pushkin-cli/tests/committed_suite.rs\n*** End Patch\n"}}"#,
agent: "codex",
hook_payload: r#"{"session_id":"gdc-09","tool_name":"apply_patch","tool_input":{"command":"*** Begin Patch\n*** Delete File: crates/pushkin-cli/tests/committed_suite.rs\n*** End Patch\n"}}"#,
daemon: "off",
expect: Expect::Agree,
};
const AUGGIE_FRAGMENT_PROTECTED: Case = Case {
name: "auggie_fragment_targeting_a_protected_path",
check_payload: r#"{"session_id":"gdc-10","tool_name":"str-replace-editor","tool_input":{"path":"pushkin.toml","new_str_1":"version = 2"}}"#,
agent: "auggie",
hook_payload: r#"{"session_id":"gdc-10","tool_name":"str-replace-editor","tool_input":{"path":"pushkin.toml","new_str_1":"version = 2"}}"#,
daemon: "off",
expect: Expect::Agree,
};
const READ_WHOLE_GATED_SOURCE: Case = Case {
name: "unbounded_read_of_a_retrieval_gated_source_file",
check_payload: r#"{"session_id":"gdc-11","tool_name":"Read","tool_input":{"file_path":"crates/pushkin-cli/src/main.rs"}}"#,
agent: "claude",
hook_payload: r#"{"session_id":"gdc-11","tool_name":"Read","tool_input":{"file_path":"crates/pushkin-cli/src/main.rs"}}"#,
daemon: "off",
expect: Expect::Diverges {
check: Decision::FailOpen,
hook: Decision::Block,
why: "F67 arm 1 — hook denies pushkin.retrieval.raw_read; check fails \
open. The read contract has no check-verb counterpart.",
},
};
const BASH_READ_GATED_SOURCE: Case = Case {
name: "shell_read_of_a_retrieval_gated_source_file",
check_payload: r#"{"session_id":"gdc-12","tool_name":"Bash","tool_input":{"command":"cat crates/pushkin-cli/src/main.rs"}}"#,
agent: "claude",
hook_payload: r#"{"session_id":"gdc-12","tool_name":"Bash","tool_input":{"command":"cat crates/pushkin-cli/src/main.rs"}}"#,
daemon: "off",
expect: Expect::Diverges {
check: Decision::FailOpen,
hook: Decision::Block,
why: "F67 arm 2 — hook denies pushkin.retrieval.raw_read via \
gate_shell_read; check has no Intent::Shell branch and fails open.",
},
};
const CORPUS: &[&Case] = &[
&WRITE_NONCONFORMING,
&WRITE_NONCONFORMING_WARM,
&WRITE_CONFORMING,
&WRITE_UNGATED,
&WRITE_PROTECTED,
&EDIT_READ_ONLY,
&MULTIEDIT_READ_ONLY,
&STOP_SWEEP,
&UNREADABLE_NAMING_PROTECTED,
&CODEX_DELETE_READ_ONLY,
&AUGGIE_FRAGMENT_PROTECTED,
&READ_WHOLE_GATED_SOURCE,
&BASH_READ_GATED_SOURCE,
];
#[test]
fn write_nonconforming_to_mapped_glob() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &WRITE_NONCONFORMING)
}
#[test]
fn write_nonconforming_to_mapped_glob_warm() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &WRITE_NONCONFORMING_WARM)
}
#[test]
fn write_conforming_to_mapped_glob() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &WRITE_CONFORMING)
}
#[test]
fn write_to_ungated_unmapped_path() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &WRITE_UNGATED)
}
#[test]
fn write_to_protected_path() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &WRITE_PROTECTED)
}
#[test]
fn edit_a_committed_read_only_test() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &EDIT_READ_ONLY)
}
#[test]
fn multiedit_a_committed_read_only_test() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &MULTIEDIT_READ_ONLY)
}
#[test]
fn stop_payload_sweeps_the_repo() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &STOP_SWEEP)
}
#[test]
fn unreadable_payload_naming_a_protected_path() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &UNREADABLE_NAMING_PROTECTED)
}
#[test]
fn codex_apply_patch_delete_of_a_read_only_test() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &CODEX_DELETE_READ_ONLY)
}
#[test]
fn auggie_fragment_targeting_a_protected_path() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &AUGGIE_FRAGMENT_PROTECTED)
}
#[test]
fn unbounded_read_of_a_retrieval_gated_source_file_diverges() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &READ_WHOLE_GATED_SOURCE)
}
#[test]
fn shell_read_of_a_retrieval_gated_source_file_diverges() -> TestResult {
let dir = repo()?;
assert_conformance(dir.path(), &BASH_READ_GATED_SOURCE)
}
#[test]
fn check_prose_and_json_agree_on_every_row() -> TestResult {
for case in CORPUS {
let dir = repo()?;
let structural = check_verdict(dir.path(), case)?;
let raw = run(
dir.path(),
&Invocation {
args: &["check"],
daemon: case.daemon,
payload: case.check_payload,
},
)?;
let parsed = violation_keys(&raw.stderr);
assert_eq!(
structural.violations, parsed,
"{}: the prose parser disagrees with the JSON envelope on the SAME \
payload, so it cannot be trusted on the hook surface. stderr={}",
case.name, raw.stderr
);
}
Ok(())
}
#[test]
fn the_hook_surface_is_not_vacuously_allowing() -> TestResult {
let dir = repo()?;
let verdict = hook_verdict(dir.path(), &WRITE_PROTECTED)?;
assert_eq!(verdict.decision, Decision::Block);
assert!(
verdict
.violations
.iter()
.any(|k| k.contains(RULE_PROTECTED)),
"hook must deny a protected-path write: {verdict:?}"
);
Ok(())
}
#[test]
fn the_corpus_covers_every_rule_it_claims() -> TestResult {
assert!(
CORPUS.len() >= 13,
"corpus rows may be added, never removed"
);
let names: Vec<&str> = CORPUS.iter().map(|c| c.name).collect();
let mut sorted = names.clone();
sorted.sort_unstable();
sorted.dedup();
assert_eq!(sorted.len(), names.len(), "corpus row names must be unique");
let dir = repo()?;
let mut seen: Vec<String> = Vec::new();
for case in CORPUS {
seen.extend(hook_verdict(dir.path(), case)?.violations);
}
for rule in [
RULE_UNVALIDATED,
RULE_PROTECTED,
RULE_READ_ONLY,
RULE_RAW_READ,
] {
assert!(
seen.iter().any(|k| k.contains(rule)),
"no corpus row exercises {rule}; the suite would agree vacuously on it"
);
}
Ok(())
}
#[test]
fn the_key_parser_rejects_ladder_decoration() {
assert_eq!(
key_from_line(
" app/api/users/route.ts:1 [contract.boundary.unvalidated_input] (contract: user)"
),
Some("app/api/users/route.ts:1 [contract.boundary.unvalidated_input]".to_owned()),
);
assert_eq!(
key_from_line(" app/api/users/route.ts:1 [contract.boundary.unvalidated_input] (contract: 'user') — attempt 1/3"),
Some("app/api/users/route.ts:1 [contract.boundary.unvalidated_input]".to_owned()),
"the check and hook renderings of one violation must key identically",
);
for decoration in [
" try: import { UserCreateSchema } from \"contracts/user.zod\"",
" fix: Parse the request body with UserCreateSchema [not a rule]",
" waiver: a human (not you) can run `pushkin waive pushkin.read_only_path`",
" nudge: the contract can be shown instead of searched — run `pushkin instructions` before retrying.",
" contract 'user' slice:",
" export const user = 1;",
"pushkin: write blocked. Fix the violations below and retry.",
"pushkin: read blocked AGAIN. Do not retry the same read — the result will be identical.",
"pushkin: STOP. This write has been blocked 3 times. Do not attempt it again.",
"",
] {
assert_eq!(
key_from_line(decoration),
None,
"decoration parsed as a violation: {decoration}"
);
}
let ladder = "pushkin: write blocked. Fix the violations below and retry.\n pushkin.toml:1 [pushkin.protected_path] — attempt 1/3\n fix: a human must make it.\n waiver: a human (not you) can run `pushkin waive pushkin.protected_path`\n app/api/users/route.ts:1 [contract.boundary.unvalidated_input] (contract: 'user') — attempt 1/3\n try: contract_show user";
assert_eq!(
violation_keys(ladder),
vec![
"app/api/users/route.ts:1 [contract.boundary.unvalidated_input]".to_owned(),
"pushkin.toml:1 [pushkin.protected_path]".to_owned(),
],
);
}