use std::path::Path;
use std::process::Command;
use std::time::Duration;
use anyhow::Result;
use super::result::{CheckResult, VerifyReport};
use crate::spawn::HELP_TIMEOUT;
#[derive(Debug, Clone)]
pub struct InvocationInput {
pub skill_md: String,
pub cli_command: Option<Vec<String>>,
pub skill_root: std::path::PathBuf,
pub spawn_cwd: std::path::PathBuf,
pub debug: bool,
}
impl InvocationInput {
pub fn new(
skill_root: &Path,
spawn_cwd: &Path,
skill_md: &str,
cli_command: Option<&[String]>,
debug: bool,
) -> Self {
Self {
skill_md: skill_md.to_string(),
cli_command: cli_command.map(<[std::string::String]>::to_vec),
skill_root: skill_root.to_path_buf(),
spawn_cwd: spawn_cwd.to_path_buf(),
debug,
}
}
}
pub fn run(input: &InvocationInput, report: &mut VerifyReport) -> Result<()> {
let skill_invocation = extract_documented_invocation(&input.skill_md);
if skill_invocation.is_none() {
report.push(CheckResult::skipped(
"invocation",
"CLI invocation drift checks",
"Skipped: pure-library project (no CLI documented in SKILL.md)",
));
return Ok(());
}
let Some(cmd) = input.cli_command.as_ref() else {
report.push(CheckResult::warn(
"invocation.not_runnable_here",
"documented CLI can be spawned for drift checks",
"SKILL.md documents a CLI invocation, but no runnable command was \
found on this machine (no built artifact / runtime missing)",
"To fix: build/install the CLI so `skillpack verify` can spawn its \
`--help`, or run verify on a machine where the CLI is installed.",
));
return Ok(());
};
if cmd.is_empty() {
report.push(CheckResult::warn(
"invocation.not_runnable_here",
"documented CLI can be spawned for drift checks",
"SKILL.md documents a CLI invocation, but the recorded command is empty",
"To fix: re-run `skillpack init` so a CLI command is recorded.",
));
return Ok(());
}
let help = run_help(cmd, &input.spawn_cwd, input.debug, report)?;
if report.has_critical_failure() {
return Ok(());
}
check_flag_drift(&help, skill_invocation.as_deref().unwrap_or(""), report);
check_subcommand_drift(cmd, &input.spawn_cwd, &input.skill_md, input.debug, report)?;
Ok(())
}
pub fn extract_documented_invocation(skill_md: &str) -> Option<String> {
if let Some(block) = heading_block(skill_md, "invocation") {
return Some(block);
}
let mut in_fence = false;
let mut block = String::new();
for line in skill_md.lines() {
let t = line.trim();
if t.starts_with("```") {
if in_fence {
if extract_flags(&block).iter().any(|f| !is_meta_flag(f)) {
return Some(block.clone());
}
block.clear();
}
in_fence = !in_fence;
continue;
}
if in_fence {
block.push_str(line);
block.push('\n');
}
}
None
}
fn heading_block(skill_md: &str, heading: &str) -> Option<String> {
let want = format!("## {heading}");
let mut in_block = false;
let mut out = String::new();
for line in skill_md.lines() {
let trimmed = line.trim();
if trimmed.starts_with("## ") || trimmed.starts_with("### ") {
if trimmed.starts_with("## ") && trimmed.eq_ignore_ascii_case(&want) {
in_block = true;
continue;
}
if in_block {
break;
}
continue;
}
if in_block {
out.push_str(line);
out.push('\n');
}
}
if in_block && !out.trim().is_empty() {
Some(out)
} else {
None
}
}
fn run_help(cmd: &[String], root: &Path, debug: bool, report: &mut VerifyReport) -> Result<String> {
let program = &cmd[0];
if debug {
eprintln!(
"[debug] spawn (cwd={}): {}{}",
root.display(),
program,
cmd[1..].iter().map(|a| format!(" {a}")).collect::<String>()
);
}
let mut c = Command::new(program);
for arg in &cmd[1..] {
c.arg(arg);
}
c.current_dir(root);
match crate::spawn::run(&mut c, HELP_TIMEOUT) {
crate::spawn::SpawnOutcome::NotFound => {
report.push(CheckResult::fail(
"invocation.help_present",
"documented CLI is installed and runnable",
format!("CLI binary `{program}` not found on PATH"),
format!(
"To fix: build/install `{program}` so it's on PATH, then re-run `skillpack verify`."
),
));
Ok(String::new())
}
crate::spawn::SpawnOutcome::SpawnFailed(e) => {
report.push(CheckResult::fail(
"invocation.help_present",
"documented CLI is installed and runnable",
format!("could not spawn `{program}`: {e}"),
"To fix: check that the binary path in skillpack.toml is correct.",
));
Ok(String::new())
}
crate::spawn::SpawnOutcome::TimedOut => {
report.push(CheckResult::fail(
"invocation.help_present",
"CLI prints `--help` quickly",
format!("`{program} --help` exceeded {}s timeout", HELP_TIMEOUT.as_secs()),
"To fix: the CLI may hang waiting on input; guard it with `</dev/null` or fix the hang before shipping.",
));
Ok(String::new())
}
crate::spawn::SpawnOutcome::RanNonZero => {
report.push(CheckResult::fail(
"invocation.help_present",
"documented `--help` exits cleanly",
format!("`{program}` returned non-zero on `--help`"),
"To fix: make `--help` exit 0, or correct the command in skillpack.toml.",
));
Ok(String::new())
}
crate::spawn::SpawnOutcome::RanClean(output) => {
if output.trim().is_empty() {
report.push(CheckResult::fail(
"invocation.help_present",
"documented `--help` produces output",
format!("`{program} --help` printed nothing"),
"To fix: implement/generate `--help` output so an agent knows the available flags.",
));
return Ok(output);
}
report.push(CheckResult::pass(
"invocation.help_present",
"documented `--help` runs and produces output",
format!("`{program}` printed {} bytes of help", output.len()),
));
Ok(output)
}
}
}
fn check_flag_drift(help_output: &str, skill_md: &str, report: &mut VerifyReport) {
let help_flags = extract_flags(help_output);
let doc_flags = extract_flags(skill_md)
.into_iter()
.filter(|f| !is_meta_flag(f))
.collect::<Vec<_>>();
if doc_flags.is_empty() {
report.push(CheckResult::warn(
"invocation.flag_drift",
"SKILL.md documents flags that match `--help`",
"no flags appear to be documented in SKILL.md (no `--flag` tokens found)",
"To fix: document the CLI's flags so an agent knows what to pass.",
));
return;
}
let mut drifted: Vec<String> = doc_flags
.iter()
.filter(|f| !help_flags.contains(*f))
.cloned()
.collect();
drifted.sort();
drifted.dedup();
if drifted.is_empty() {
report.push(CheckResult::pass(
"invocation.flag_drift",
"every documented flag exists in `--help`",
format!(
"all {} documented flag(s) present in --help",
doc_flags.len()
),
));
} else {
let first = &drifted[0];
let line_hint = skill_md
.lines()
.position(|l| l.contains(first.as_str()))
.map(|n| n + 1);
let mut fail = CheckResult::fail(
"invocation.flag_drift",
"every documented flag exists in `--help`",
format!(
"documented flag(s) missing from `--help`: {}",
drifted.join(", ")
),
format!(
"To fix: remove `{first}` from SKILL.md, or add `{first}` to your CLI's `--help`."
),
);
fail.location = Some(("SKILL.md".to_string(), line_hint));
report.push(fail);
}
reverse_drift(&help_flags, &doc_flags, report);
}
fn reverse_drift(help_flags: &[String], doc_flags: &[String], report: &mut VerifyReport) {
let mut undocumented: Vec<String> = help_flags
.iter()
.filter(|f| !is_meta_flag(f) && !doc_flags.contains(f))
.cloned()
.collect();
undocumented.sort();
undocumented.dedup();
if undocumented.is_empty() {
return;
}
report.push(CheckResult::warn(
"invocation.undocumented_flags",
"every `--help` flag is documented in SKILL.md",
format!(
"`--help` advertises flags the skill doesn't document: {}",
undocumented.join(", ")
),
"To fix: document these flags in SKILL.md so an agent knows it can pass them.",
));
}
pub fn is_meta_flag(flag: &str) -> bool {
matches!(flag, "--help" | "-h" | "--version" | "-V" | "--help-all")
}
pub fn extract_subcommands(help_output: &str) -> Vec<String> {
let mut out = Vec::new();
let mut in_section = false;
for line in help_output.lines() {
let trimmed = line.trim();
let is_header = matches!(
trimmed.to_ascii_lowercase().as_str(),
"commands:" | "subcommands:"
);
if is_header {
in_section = true;
continue;
}
if !in_section {
continue;
}
if trimmed.is_empty() {
break;
}
if line == trimmed {
break;
}
let Some(name) = trimmed.split_whitespace().next() else {
continue;
};
if name == "help" {
continue;
}
if !out.contains(&name.to_string()) {
out.push(name.to_string());
}
}
out
}
pub fn extract_documented_subcommands(skill_md: &str) -> Vec<String> {
let want = "### Subcommands";
let mut in_block = false;
let mut block = String::new();
for line in skill_md.lines() {
let trimmed = line.trim();
if trimmed.starts_with("### ") || trimmed.starts_with("## ") {
if in_block {
break;
}
if trimmed.eq_ignore_ascii_case(want) {
in_block = true;
}
continue;
}
if in_block {
block.push_str(line);
block.push('\n');
}
}
let mut out = Vec::new();
for line in block.lines() {
let trimmed = line.trim();
if !trimmed.starts_with("- ") {
continue;
}
let after = trimmed.strip_prefix("- ").unwrap_or(trimmed);
let name = after
.split('`')
.nth(1)
.map(str::trim)
.filter(|s| !s.is_empty());
if let Some(n) = name {
if !out.contains(&n.to_string()) {
out.push(n.to_string());
}
}
}
out
}
fn spawn_capture(cmd: &[String], root: &Path, timeout: Duration) -> Option<String> {
let mut c = Command::new(&cmd[0]);
for arg in &cmd[1..] {
c.arg(arg);
}
c.current_dir(root);
match crate::spawn::run(&mut c, timeout) {
crate::spawn::SpawnOutcome::RanClean(out) => Some(out),
_ => None,
}
}
fn check_subcommand_drift(
base_cmd: &[String],
spawn_cwd: &Path,
skill_md: &str,
debug: bool,
report: &mut VerifyReport,
) -> Result<()> {
let documented = extract_documented_subcommands(skill_md);
if documented.is_empty() {
return Ok(());
}
let mut base = base_cmd.to_vec();
if base.last().is_some_and(|t| t == "--help") {
base.pop();
}
for sub in &documented {
let mut cmd = base.clone();
cmd.push(sub.clone());
cmd.push("--help".to_string());
if debug {
eprintln!(
"[debug] spawn (cwd={}): {}",
spawn_cwd.display(),
cmd.join(" ")
);
}
let captured = spawn_capture(&cmd, spawn_cwd, HELP_TIMEOUT);
let Some(help) = captured else {
report.push(CheckResult::fail(
"invocation.subcommand_drift",
"every documented subcommand can be spawned for drift checks",
format!("documented subcommand `{sub}` could not be spawned for `--help` (missing runtime / non-zero exit / timeout)"),
"To fix: build/install the CLI so the subcommand is runnable, or remove the subcommand from SKILL.md.",
));
continue;
};
let bullet = subcommand_bullet(skill_md, sub);
let doc_flags: Vec<String> = extract_flags(&bullet)
.into_iter()
.filter(|f| !is_meta_flag(f))
.collect();
let help_flags = extract_flags(&help);
let drifted: Vec<String> = doc_flags
.iter()
.filter(|f| !help_flags.contains(*f))
.cloned()
.collect();
let check_name = format!("documented subcommand `{sub}` flags match `--help`");
if drifted.is_empty() {
report.push(CheckResult::pass(
"invocation.subcommand_drift",
&check_name,
format!("`{sub}` documented flags all present in --help"),
));
} else {
report.push(CheckResult::fail(
"invocation.subcommand_drift",
&check_name,
format!("subcommand `{sub}` documents flags missing from `--help`: {}", drifted.join(", ")),
format!("To fix: remove the flags from SKILL.md's `{sub}` bullet, or add them to `{sub}`'s `--help`."),
));
}
let undocumented: Vec<String> = help_flags
.iter()
.filter(|f| !is_meta_flag(f) && !doc_flags.contains(*f))
.cloned()
.collect();
if !undocumented.is_empty() {
let warn_name = format!("subcommand `{sub}` advertises no undocumented flags");
report.push(CheckResult::warn(
"invocation.subcommand_drift",
&warn_name,
format!(
"`{sub} --help` advertises flags the skill doesn't document: {}",
undocumented.join(", ")
),
"To fix: document these flags in SKILL.md's `{sub}` bullet.",
));
}
}
Ok(())
}
fn subcommand_bullet(skill_md: &str, sub: &str) -> String {
let needle = format!("- `{sub}`");
skill_md
.lines()
.find(|l| l.trim().starts_with(&needle))
.unwrap_or_default()
.to_string()
}
pub fn extract_flags(text: &str) -> Vec<String> {
let mut out = Vec::new();
for tok in text.split_whitespace() {
let t = if let Some(i) = tok.find("[=") {
tok[..i].to_string()
} else {
tok.to_string()
};
let t = t
.trim_matches(|c: char| c.is_ascii_punctuation() && c != '-')
.to_string();
if !t.starts_with('-') || t.len() < 2 {
continue;
}
let first_letter = match t.chars().find(|c| *c != '-') {
Some(c) => c,
None => continue,
};
if !first_letter.is_ascii_alphabetic() {
continue;
}
if t.contains('/') || t.contains('\'') {
continue;
}
let dash_count = t.chars().take_while(|c| *c == '-').count();
if dash_count == 1 && t.len() > 2 {
continue;
}
let flag: String = t
.split('=')
.next()
.unwrap_or(&t)
.trim_end_matches([',', '.', ';', ':', ')', ']', '\''])
.to_string();
if flag.len() >= 2 && !out.contains(&flag) {
out.push(flag);
}
}
out
}
#[cfg(test)]
mod checks {
use super::*;
#[test]
fn extracts_double_and_single_flags() {
let f = extract_flags("Usage: foo --bar -x --baz=42 end");
assert!(f.contains(&"--bar".to_string()));
assert!(f.contains(&"--baz".to_string()));
assert!(f.contains(&"-x".to_string()));
assert!(!f.iter().any(|s| s == "Usage:"));
}
#[test]
fn ignores_hyphenated_prose() {
let f = extract_flags("a two-step process - and dash-2 numbers");
assert!(!f.iter().any(|s| s == "-2"));
assert!(!f.iter().any(|s| s == "two-step"));
}
#[test]
fn strips_clap_optional_arg_suffix_consistently() {
assert_eq!(
extract_flags("--hyperlink[=<when>]"),
vec!["--hyperlink".to_string()]
);
assert_eq!(
extract_flags("`--hyperlink`"),
vec!["--hyperlink".to_string()]
);
assert_eq!(
extract_flags("--strip-cwd-prefix[=<when>]"),
vec!["--strip-cwd-prefix".to_string()]
);
}
#[test]
fn ignores_prose_examples_in_help_text() {
let f = extract_flags("fd -tf -tl -tx -te -td");
assert!(f.is_empty(), "multi-char short flags from prose: got {f:?}");
let f = extract_flags("fd -- '-foo' pattern");
assert!(
!f.iter().any(|s| s == "-foo"),
"example pattern leaked: got {f:?}"
);
let f = extract_flags("place the -x'/\u{27}--exec option last");
assert!(f.is_empty(), "prose separators leaked: got {f:?}");
let f = extract_flags("Comparable to the -mount or -xdev filters of find(1)");
assert!(f.is_empty(), "find(1) prose leaked: got {f:?}");
}
#[test]
fn documented_invocation_from_heading() {
let skill = "---\nname: foo\n---\n\n## Invocation\n\n```\nfoo --new\n```\n";
let block = extract_documented_invocation(skill).expect("heading block");
assert!(block.contains("foo --new"));
assert!(extract_flags(&block).contains(&"--new".to_string()));
}
#[test]
fn documented_invocation_from_fenced_flags_for_handwritten_skill() {
let skill = "---\nname: sample-broken\n---\n\n# sample-broken\n\n```\nsample-broken --nonexistent --new\n```\n";
assert!(extract_documented_invocation(skill).is_some());
}
#[test]
fn documented_invocation_none_for_pure_library() {
let skill = "---\nname: x\n---\n\n## Usage\n\n```\nimport { parse } from 'fastcsv'\n```\n";
assert!(extract_documented_invocation(skill).is_none());
}
#[test]
fn reverse_drift_warns_on_undocumented_help_flag() {
let mut report = super::super::result::VerifyReport::default();
reverse_drift(
&["--new".to_string(), "--verbose".to_string()],
&["--new".to_string()],
&mut report,
);
assert_eq!(report.results.len(), 1);
assert_eq!(
report.results[0].severity,
super::super::result::Severity::Warn
);
assert!(report.results[0].message.contains("--verbose"));
}
#[test]
fn extract_subcommands_from_skillpack_help() {
let help = "\
Generate and verify the agent-distribution layer for any OSS project.
Usage: skillpack [OPTIONS] <COMMAND>
Commands:
init Scaffold the distribution layer
verify Check the distribution files against the schema
help Print this message or the help of the given subcommand(s)
Options:
--verbose Print what skillpack detected in the repo
-h, --help Print help
-V, --version Print version
";
assert_eq!(
extract_subcommands(help),
vec!["init".to_string(), "verify".to_string()]
);
}
#[test]
fn extract_subcommands_empty_for_non_subcommand_help() {
assert_eq!(
extract_subcommands("Usage: chronicle [--new <entry>] [--verbose]"),
Vec::<String>::new()
);
assert_eq!(extract_subcommands(""), Vec::<String>::new());
}
#[test]
fn extract_subcommands_stops_at_blank_gap() {
let help = "\
Usage: x <COMMAND>
Commands:
foo one
bar two
Options:
--global g
";
assert_eq!(
extract_subcommands(help),
vec!["foo".to_string(), "bar".to_string()]
);
}
#[test]
fn extract_documented_subcommands_from_skill_bullets() {
let skill = "\
# x
## Invocation
```
x --new
```
### Subcommands
- `init` — flags: `--root`, `--non-interactive`
- `verify` — flags: `--format`
";
assert_eq!(
extract_documented_subcommands(skill),
vec!["init".to_string(), "verify".to_string()]
);
}
#[test]
fn extract_documented_subcommands_empty_when_no_block() {
let skill = "## Invocation\n\n```\nchronicle --new\n```\n";
assert!(extract_documented_subcommands(skill).is_empty());
}
}