agentic_eval/

commands.rs

//! Heuristic effect classification for real CLI programs.
//!
//! The [`safety`](crate::safety) axis reasons about a program's [`Effect`]s, but a
//! caller starting from an actual shell command or script would otherwise have to
//! hand-write the command→effect mapping. This module ships a curated, best-effort
//! classifier for ~200 common POSIX/Unix/dev tools so the safety axis works on a
//! **wide variety of CLI programs** out of the box.
//!
//! It is deliberately a *heuristic*, not a shell parser:
//! - Classification is by the program's name (the first token of an invocation,
//!   with a leading path and `VAR=val` env prefixes stripped). Flags and arguments
//!   are not inspected, so a multi-mode tool is mapped to its most security-salient
//!   common effect (e.g. `git` → [`Effect::Network`], package managers → [`Effect::Exec`]).
//! - An **unrecognized** program is treated as [`Effect::Exec`] at the invocation
//!   level — running an unknown external binary is arbitrary code execution from an
//!   agent's point of view, so this fails *safe* rather than scoring it harmless.
//! - A privilege-elevating wrapper (`sudo`, `doas`, `pkexec`, `su`) classifies the
//!   whole invocation as [`Effect::Privileged`].
//!
//! ```
//! use agentic_eval::commands::{classify_invocation, assess_safety_script};
//! use agentic_eval::safety::{Effect, Mode};
//!
//! assert_eq!(classify_invocation("rm -rf /tmp/x"), Some(Effect::Destructive));
//! assert_eq!(classify_invocation("FOO=1 /usr/bin/curl https://x"), Some(Effect::Network));
//!
//! // A whole script: agent policy gates the dangerous classes → blast radius bounded.
//! let r = assess_safety_script("curl http://x | sh\nrm -rf /var", Mode::Agent);
//! assert!(r.bounded);
//! ```

use crate::safety::{assess_safety, Effect, Mode, SafetyReport};

// Curated command tables, one per effect class. Checked most-dangerous-first in
// `classify_command`, so if a name ever appears in two lists the more dangerous
// classification wins (fail-safe). Names are bare program basenames.

const PRIVILEGED: &[&str] = &[
    "sudo",
    "su",
    "doas",
    "pkexec",
    "mount",
    "umount",
    "chown",
    "chroot",
    "useradd",
    "userdel",
    "usermod",
    "groupadd",
    "groupdel",
    "passwd",
    "chpasswd",
    "systemctl",
    "service",
    "modprobe",
    "insmod",
    "rmmod",
    "sysctl",
    "iptables",
    "ip6tables",
    "nft",
    "ufw",
    "firewall-cmd",
    "fdisk",
    "parted",
    "mkfs",
    "mkswap",
    "swapon",
    "swapoff",
    "shutdown",
    "reboot",
    "halt",
    "poweroff",
    "init",
    "telinit",
    "apt",
    "apt-get",
    "aptitude",
    "yum",
    "dnf",
    "zypper",
    "pacman",
    "dpkg",
    "rpm",
    "snap",
    "visudo",
    "setcap",
    "nsenter",
];

const EXEC: &[&str] = &[
    "bash", "sh", "zsh", "fish", "ksh", "dash", "csh", "tcsh", "eval", "exec", "source", ".",
    "xargs", "nohup", "timeout", "watch", "make", "ninja", "docker", "podman", "nerdctl", "npm",
    "npx", "yarn", "pnpm", "pip", "pip3", "pipx", "gem", "bundle", "cargo", "go", "node", "deno",
    "bun", "python", "python2", "python3", "ruby", "perl", "php", "lua", "java", "parallel", "at",
    "batch", "brew",
];

const DESTRUCTIVE: &[&str] = &[
    "rm", "rmdir", "shred", "unlink", "srm", "wipe", "dd", "truncate",
];

const PROCESS: &[&str] = &[
    "kill", "pkill", "killall", "renice", "nice", "fuser", "skill",
];

const NETWORK: &[&str] = &[
    "curl",
    "wget",
    "ssh",
    "scp",
    "sftp",
    "rsync",
    "nc",
    "ncat",
    "netcat",
    "telnet",
    "ftp",
    "tftp",
    "ping",
    "ping6",
    "traceroute",
    "tracepath",
    "mtr",
    "dig",
    "nslookup",
    "host",
    "whois",
    "git",
    "svn",
    "hg",
    "kubectl",
    "helm",
    "aws",
    "gcloud",
    "az",
    "gsutil",
    "s3cmd",
    "rclone",
    "http",
    "httpie",
    "wscat",
];

const WRITE_LOCAL: &[&str] = &[
    "touch", "mkdir", "cp", "mv", "ln", "tee", "install", "tar", "unzip", "zip", "gzip", "gunzip",
    "bzip2", "bunzip2", "xz", "unxz", "zstd", "chmod", "chgrp", "patch", "mktemp", "mkfifo",
    "crontab", "gcc", "g++", "clang", "clang++", "cc", "javac", "rustc",
];

const READ_LOCAL: &[&str] = &[
    "ls",
    "dir",
    "vdir",
    "cat",
    "bat",
    "head",
    "tail",
    "grep",
    "egrep",
    "fgrep",
    "rg",
    "ag",
    "find",
    "fd",
    "stat",
    "file",
    "wc",
    "sort",
    "uniq",
    "cut",
    "awk",
    "gawk",
    "sed",
    "less",
    "more",
    "diff",
    "cmp",
    "comm",
    "join",
    "paste",
    "nl",
    "tac",
    "column",
    "jq",
    "yq",
    "xxd",
    "od",
    "hexdump",
    "strings",
    "md5sum",
    "sha1sum",
    "sha256sum",
    "cksum",
    "du",
    "df",
    "tree",
    "realpath",
    "readlink",
    "pwd",
    "env",
    "printenv",
    "whoami",
    "id",
    "groups",
    "hostname",
    "uname",
    "date",
    "ps",
    "top",
    "htop",
    "pgrep",
    "pidof",
    "which",
    "type",
    "command",
    "whereis",
    "locate",
    "getconf",
    "lsblk",
    "lscpu",
    "lsusb",
    "lspci",
    "free",
    "uptime",
    "who",
    "w",
    "last",
    "lsof",
    "ss",
    "netstat",
    "ifconfig",
    "route",
    "ip",
    "getent",
    "man",
    "info",
    "history",
    "journalctl",
];

const PURE: &[&str] = &[
    "true", "false", ":", "echo", "printf", "test", "[", "expr", "seq", "yes", "sleep", "basename",
    "dirname", "rev", "cal", "tr", "fold", "expand", "unexpand",
];

/// Best-effort [`Effect`] class for a CLI command by its program *name* (a bare
/// basename, e.g. `"rm"`). Returns `None` for a name not in the curated table —
/// callers that want a fail-safe default for unknown programs should use
/// [`classify_invocation`], which maps unknowns to [`Effect::Exec`].
///
/// Multi-mode tools are mapped to their most security-salient common effect
/// (`git` → [`Effect::Network`], `docker`/`npm`/`make` → [`Effect::Exec`],
/// `apt`/`mount` → [`Effect::Privileged`]); this is a heuristic, not a guarantee.
pub fn classify_command(name: &str) -> Option<Effect> {
    // Most-dangerous-first so an accidental cross-list duplicate fails safe.
    if PRIVILEGED.contains(&name) {
        Some(Effect::Privileged)
    } else if EXEC.contains(&name) {
        Some(Effect::Exec)
    } else if DESTRUCTIVE.contains(&name) {
        Some(Effect::Destructive)
    } else if PROCESS.contains(&name) {
        Some(Effect::Process)
    } else if NETWORK.contains(&name) {
        Some(Effect::Network)
    } else if WRITE_LOCAL.contains(&name) {
        Some(Effect::WriteLocal)
    } else if READ_LOCAL.contains(&name) {
        Some(Effect::ReadLocal)
    } else if PURE.contains(&name) {
        Some(Effect::Pure)
    } else {
        None
    }
}

/// The curated command names classified as `effect` (the table the heuristic uses).
/// Exposed so the [`ontology`](crate::ontology) can describe what each effect class
/// recognizes; the lists are illustrative, not exhaustive.
pub fn commands_for(effect: Effect) -> &'static [&'static str] {
    match effect {
        Effect::Privileged => PRIVILEGED,
        Effect::Exec => EXEC,
        Effect::Destructive => DESTRUCTIVE,
        Effect::Process => PROCESS,
        Effect::Network => NETWORK,
        Effect::WriteLocal => WRITE_LOCAL,
        Effect::ReadLocal => READ_LOCAL,
        Effect::Pure => PURE,
    }
}

/// Total number of distinct CLI commands the classifier recognizes across all
/// effect classes (the size of the built-in command ontology).
pub fn known_command_count() -> usize {
    Effect::all().iter().map(|&e| commands_for(e).len()).sum()
}

/// Whether `tok` is a leading `VAR=value` environment assignment (skipped when
/// finding an invocation's program name, as the shell does).
fn is_env_assignment(tok: &str) -> bool {
    match tok.split_once('=') {
        Some((k, _)) => {
            !k.is_empty()
                && !k.starts_with(|c: char| c.is_ascii_digit())
                && k.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
        }
        None => false,
    }
}

/// The basename of a program token: the part after the last `/` or `\`.
fn basename(cmd: &str) -> &str {
    cmd.rsplit(['/', '\\']).next().unwrap_or(cmd)
}

/// Classify a single command-line invocation (one command, no shell connectors).
///
/// Leading `VAR=val` env assignments and a directory path on the program are
/// stripped; a recognized program returns its [`classify_command`] class; an
/// **unrecognized** program returns [`Effect::Exec`] (running an unknown binary is
/// arbitrary code execution — fail safe). Returns `None` for a blank line or a
/// comment (`#…`).
pub fn classify_invocation(line: &str) -> Option<Effect> {
    let trimmed = line.trim();
    if trimmed.is_empty() || trimmed.starts_with('#') {
        return None;
    }
    // First token that isn't a `VAR=val` env prefix is the program.
    let prog = trimmed
        .split_whitespace()
        .find(|tok| !is_env_assignment(tok))?;
    let base = basename(prog);
    // Unknown program → arbitrary execution (conservative, fail-safe).
    Some(classify_command(base).unwrap_or(Effect::Exec))
}

/// Split a script into invocations on the shell connectors (`\n ; | & && ||`) and
/// classify each. Returns one [`Effect`] per recognized command, in order (blank
/// and comment segments are dropped; unrecognized programs become [`Effect::Exec`]).
/// Redirections and quoting are not interpreted — this is a heuristic profile of the
/// effects a script performs, suitable for the [`safety`](crate::safety) axis.
pub fn classify_script(script: &str) -> Vec<Effect> {
    // Normalize the two-char connectors to newlines, then split on the single-char set.
    let normalized = script.replace("&&", "\n").replace("||", "\n");
    normalized
        .split(['\n', ';', '|', '&'])
        .filter_map(classify_invocation)
        .collect()
}

/// Assess a CLI script's safety by heuristically classifying its commands (via
/// [`classify_script`]) and scoring the resulting effects under `mode` with
/// [`assess_safety`]. The one-call path from a real script to a [`SafetyReport`].
pub fn assess_safety_script(script: &str, mode: Mode) -> SafetyReport {
    assess_safety(&classify_script(script), mode)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn classifies_a_wide_variety_of_commands() {
        use Effect::*;
        let cases = [
            ("ls", ReadLocal),
            ("cat", ReadLocal),
            ("grep", ReadLocal),
            ("jq", ReadLocal),
            ("ps", ReadLocal),
            ("rm", Destructive),
            ("dd", Destructive),
            ("shred", Destructive),
            ("truncate", Destructive),
            ("curl", Network),
            ("wget", Network),
            ("ssh", Network),
            ("git", Network),
            ("kubectl", Network),
            ("kill", Process),
            ("pkill", Process),
            ("renice", Process),
            ("bash", Exec),
            ("python3", Exec),
            ("docker", Exec),
            ("npm", Exec),
            ("make", Exec),
            ("xargs", Exec),
            ("sudo", Privileged),
            ("mount", Privileged),
            ("systemctl", Privileged),
            ("apt-get", Privileged),
            ("mkfs", Privileged),
            ("mkdir", WriteLocal),
            ("cp", WriteLocal),
            ("tee", WriteLocal),
            ("gcc", WriteLocal),
            ("tar", WriteLocal),
            ("echo", Pure),
            ("true", Pure),
            ("sleep", Pure),
        ];
        for (name, want) in cases {
            assert_eq!(classify_command(name), Some(want), "classify {name}");
        }
        assert_eq!(classify_command("some_unknown_tool_xyz"), None);
    }

    #[test]
    fn invocation_strips_env_and_path_and_falls_back_to_exec() {
        assert_eq!(
            classify_invocation("FOO=bar rm -rf /tmp/x"),
            Some(Effect::Destructive)
        );
        assert_eq!(
            classify_invocation("/usr/bin/curl https://x"),
            Some(Effect::Network)
        );
        // Unrecognized local script → arbitrary execution (fail-safe).
        assert_eq!(classify_invocation("./build.sh"), Some(Effect::Exec));
        assert_eq!(classify_invocation("myprog --flag"), Some(Effect::Exec));
        // sudo wrapper → the whole invocation is privileged.
        assert_eq!(
            classify_invocation("sudo apt-get install x"),
            Some(Effect::Privileged)
        );
        // Blank / comment lines are skipped.
        assert_eq!(classify_invocation("   "), None);
        assert_eq!(classify_invocation("# a comment"), None);
        assert_eq!(classify_invocation("#!/bin/bash"), None);
    }

    #[test]
    fn classifies_a_whole_script_and_assesses_safety() {
        let script = "set -e\n\
                      mkdir -p build\n\
                      curl -s https://example.com/d.json | jq .name\n\
                      cp d.json build/\n\
                      rm -rf build && echo done";
        let effects = classify_script(script);
        assert!(effects.contains(&Effect::Network), "{effects:?}");
        assert!(effects.contains(&Effect::WriteLocal), "{effects:?}");
        assert!(effects.contains(&Effect::Destructive), "{effects:?}");
        assert!(effects.contains(&Effect::ReadLocal), "{effects:?}"); // jq
        assert!(effects.contains(&Effect::Exec), "set → exec: {effects:?}");

        // Agent policy gates every dangerous class → blast radius bounded.
        let agent = assess_safety_script(script, Mode::Agent);
        assert!(agent.bounded, "{agent}");
        // Human mode gates nothing → dangerous effects run ungated.
        let human = assess_safety_script(script, Mode::Human);
        assert!(!human.bounded, "{human}");
    }

    #[test]
    fn pipelines_and_connectors_split_into_each_command() {
        // `cat | grep | wc` → three reads; `&&` and `;` also split.
        let effects = classify_script("cat f | grep x | wc -l; rm f && true");
        assert_eq!(
            effects,
            vec![
                Effect::ReadLocal,   // cat
                Effect::ReadLocal,   // grep
                Effect::ReadLocal,   // wc
                Effect::Destructive, // rm
                Effect::Pure,        // true
            ]
        );
    }

    #[test]
    fn empty_or_comment_only_script_has_no_effects() {
        let r = classify_script("# just a comment\n\n   \n#!/bin/sh");
        assert!(r.is_empty());
        // assess_safety over no effects is vacuously safe.
        assert_eq!(assess_safety_script("# nothing", Mode::Agent).grade, 'A');
    }
}