eval_magic/adapters/
skill_shadow.rs

1//! Harness-neutral skill-shadow report types and formatters.
2//!
3//! A *shadow* is a staged skill name that is also discoverable from the
4//! operator's live environment, contaminating the with/without comparison.
5//! Detection is harness-specific (each adapter's
6//! [`detect_shadowed_skills`](crate::adapters::HarnessAdapter::detect_shadowed_skills)
7//! decides what "discoverable" means); the report shape and its renderings are
8//! shared. The remediation text is Claude-flavored because only the Claude Code
9//! adapter produces reports today — promote the formatters to adapter methods
10//! when a second harness wires this preflight.
11
12use serde::{Deserialize, Serialize};
13
14const ISOLATION_DOC: &str = "docs/claude-notes.md → \"Isolating from installed plugins\"";
15
16/// A staged skill that is also discoverable from the live environment.
17#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
18#[serde(tag = "kind", rename_all = "kebab-case")]
19pub enum ShadowSource {
20    Plugin {
21        plugin: String,
22        skill_name: String,
23        path: String,
24    },
25    GlobalSkill {
26        skill_name: String,
27        path: String,
28    },
29}
30
31impl ShadowSource {
32    pub(crate) fn skill_name(&self) -> &str {
33        match self {
34            ShadowSource::Plugin { skill_name, .. } => skill_name,
35            ShadowSource::GlobalSkill { skill_name, .. } => skill_name,
36        }
37    }
38
39    fn source_label(&self) -> String {
40        match self {
41            ShadowSource::Plugin { plugin, .. } => format!("enabled plugin '{plugin}'"),
42            ShadowSource::GlobalSkill { .. } => "the global skills dir".to_string(),
43        }
44    }
45}
46
47/// The detector's findings for a run.
48#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
49pub struct PluginShadowReport {
50    pub config_dir: String,
51    pub shadowed: Vec<ShadowSource>,
52}
53
54/// One `validity_warnings` line per shadowed skill (for benchmark.json).
55pub fn shadow_validity_warnings(report: &PluginShadowReport) -> Vec<String> {
56    report
57        .shadowed
58        .iter()
59        .map(|s| {
60            format!(
61                "staged skill '{}' is also provided by {} — each claude -p dispatch could discover \
62                 both copies, so with/without results may be contaminated. Isolate each dispatch's \
63                 Claude config (see {}).",
64                s.skill_name(),
65                s.source_label(),
66                ISOLATION_DOC
67            )
68        })
69        .collect()
70}
71
72/// Build-time banner for the runner. Empty string when nothing is shadowed.
73pub fn format_shadow_banner(report: &PluginShadowReport) -> String {
74    if report.shadowed.is_empty() {
75        return String::new();
76    }
77    let mut lines = vec![
78        String::new(),
79        "⚠ Plugin-shadow warning: skills staged for this eval are ALSO discoverable".to_string(),
80        "  from your live environment:".to_string(),
81    ];
82    for s in &report.shadowed {
83        lines.push(format!("  • {} — {}", s.skill_name(), s.source_label()));
84    }
85    lines.push(
86        "  Each `claude -p` dispatch loads your user/global plugins and skills, so".to_string(),
87    );
88    lines.push("  both the staged copy and the installed copy are discoverable — the".to_string());
89    lines.push(
90        "  with/without comparison may be contaminated and the control arm is not truly"
91            .to_string(),
92    );
93    lines.push(
94        "  skill-absent. The runner cannot strip an installed plugin from the dispatch."
95            .to_string(),
96    );
97    lines.push("  Isolate each dispatch's Claude config, one of these ways:".to_string());
98    lines.push(
99        "  1. Drop user-scope plugins, keep auth: add `--setting-sources project,local`"
100            .to_string(),
101    );
102    lines.push("     to each dispatch.".to_string());
103    lines.push(
104        "  2. Disable the specific plugin: set `\"enabledPlugins\": { \"<plugin>@<marketplace>\":"
105            .to_string(),
106    );
107    lines.push("     false }` in a settings source the dispatch loads.".to_string());
108    lines.push("  3. Clean config dir (strips everything): run each dispatch under".to_string());
109    lines.push(
110        "     `CLAUDE_CONFIG_DIR=\"$(mktemp -d)\"` — auth may need `ANTHROPIC_API_KEY`."
111            .to_string(),
112    );
113    lines.push(format!("  Full mechanics: {ISOLATION_DOC}."));
114    lines.join("\n")
115}
116
117#[cfg(test)]
118mod tests {
119    use super::*;
120
121    fn sample_report() -> PluginShadowReport {
122        PluginShadowReport {
123            config_dir: "/x".into(),
124            shadowed: vec![ShadowSource::Plugin {
125                plugin: "slow-powers@slowdini".into(),
126                skill_name: "verification-before-completion".into(),
127                path: "/p".into(),
128            }],
129        }
130    }
131
132    #[test]
133    fn validity_warnings_name_skill_plugin_and_contamination() {
134        let warnings = shadow_validity_warnings(&sample_report());
135        assert_eq!(warnings.len(), 1);
136        assert!(warnings[0].contains("verification-before-completion"));
137        assert!(warnings[0].contains("slow-powers@slowdini"));
138        assert!(warnings[0].to_lowercase().contains("contaminat"));
139    }
140
141    #[test]
142    fn banner_is_empty_when_nothing_shadowed() {
143        let empty = PluginShadowReport {
144            config_dir: "/x".into(),
145            shadowed: vec![],
146        };
147        assert_eq!(format_shadow_banner(&empty), "");
148    }
149
150    #[test]
151    fn banner_lists_shadowed_skills_and_points_at_isolation_docs() {
152        let banner = format_shadow_banner(&sample_report());
153        assert!(banner.contains("verification-before-completion"));
154        assert!(banner.contains("slow-powers@slowdini"));
155        assert!(banner.to_lowercase().contains("isolat"));
156    }
157
158    #[test]
159    fn banner_carries_the_three_isolation_recipes_inline() {
160        // The remediation options live in the banner itself — shown exactly when
161        // the contamination is detected — not in a README section the operator
162        // has to go find.
163        let banner = format_shadow_banner(&sample_report());
164        assert!(
165            banner.contains("--setting-sources project,local"),
166            "banner names the setting-sources option: {banner}"
167        );
168        assert!(
169            banner.contains("enabledPlugins"),
170            "banner names the per-plugin disable option: {banner}"
171        );
172        assert!(
173            banner.contains("CLAUDE_CONFIG_DIR"),
174            "banner names the clean-config-dir option: {banner}"
175        );
176        assert!(
177            banner.contains("docs/claude-notes.md"),
178            "banner points at the harness dev notes for the full mechanics: {banner}"
179        );
180    }
181
182    #[test]
183    fn validity_warnings_point_at_a_doc_that_exists() {
184        let warnings = shadow_validity_warnings(&sample_report());
185        assert!(
186            warnings[0].contains("docs/claude-notes.md"),
187            "warning points at the harness dev notes: {}",
188            warnings[0]
189        );
190    }
191}
eval_magic/adapters/skill_shadow.rs

eval_magic/adapters/
skill_shadow.rs
